diff --git a/.project-metadata.yaml b/.project-metadata.yaml index 85b9b1b..111e4b4 100644 --- a/.project-metadata.yaml +++ b/.project-metadata.yaml @@ -69,7 +69,7 @@ tasks: script: build/build_client.py arguments: None cpu: 2 - memory: 2 + memory: 4 short_summary: Create job to build client application environment: TASK_TYPE: CREATE/RUN_JOB diff --git a/app/client/eslint.config.js b/app/client/eslint.config.js index 092408a..6a991c7 100644 --- a/app/client/eslint.config.js +++ b/app/client/eslint.config.js @@ -23,6 +23,7 @@ export default tseslint.config( 'warn', { allowConstantExport: true }, ], + '@typescript-eslint/no-explicit-any': ['warn', { 'fixToUnknown': true, 'ignoreRestArgs': false }] }, }, ) diff --git a/app/client/package.json b/app/client/package.json index f3e0095..bf69255 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -16,6 +16,8 @@ "@mui/icons-material": "6.1.7", "@mui/material": "6.1.7", "@tanstack/react-query": "5.66.0", + "ag-grid-community": "33.2.4", + "ag-grid-react":"33.2.4", "antd": "5.22.1", "axios": "1.6.7", "lodash": "4.17.21", diff --git a/app/client/src/api/api.ts b/app/client/src/api/api.ts index 19d9073..b343957 100644 --- a/app/client/src/api/api.ts +++ b/app/client/src/api/api.ts @@ -27,8 +27,11 @@ export const useFetchModels = (): UseFetchApiReturn => { return useFetch(url); } -export const useFetchDefaultPrompt = (useCase: string): UseFetchApiReturn => { - const url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`; +export const useFetchDefaultPrompt = (useCase: string, workflowType?: WorkerType): UseFetchApiReturn => { + let url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`; + if (workflowType && workflowType === 'freeform') { + url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_freeform_prompt`; + } return useFetch(url); } @@ -42,7 +45,7 @@ export const useFetchDefaultModelParams = (): UseFetchApiReturn() => { - const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/generate`; +export const useTriggerDatagen = (workflow_type: string) => { + const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/${workflow_type === 'freeform' ? 'freeform' : 'generate'}`; return usePostApi(genDatasetUrl); } diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index 2db6fd4..d93c70c 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -9,6 +9,8 @@ import { MODEL_PROVIDER_LABELS } from './constants'; import { ModelProviders, ModelProvidersDropdownOpts } from './types'; import { useWizardCtx } from './utils'; import FileSelectorButton from './FileSelectorButton'; +import first from 'lodash/first'; +import get from 'lodash/get'; const StepContainer = styled(Flex)` background: white; @@ -31,7 +33,8 @@ export const USECASE_OPTIONS = [ export const WORKFLOW_OPTIONS = [ { label: 'Supervised Fine-Tuning', value: 'supervised-fine-tuning' }, - { label: 'Custom Data Generation', value: 'custom' } + { label: 'Custom Data Generation', value: 'custom' }, + { label: 'Freefrom Data Generation', value: 'freeform' } ]; export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ @@ -65,6 +68,13 @@ const Configure = () => { validateForm() }, [form, formData]) + // keivan + useEffect(() => { + if (formData && formData?.inference_type === undefined) { + form.setFieldValue('inference_type', ModelProviders.CAII); + } + }, [formData]); + const labelCol = { span: 8 }; @@ -106,7 +116,16 @@ const Configure = () => { setSelectedFiles([]); } } + + const onAddExampleFiles = (files: File[]) => { + console.log('onAddExampleFiles', files); + if (!isEmpty(files)) { + const file = first(files); + form.setFieldValue('example_path', get(file, '_path')); + } + } + console.log('formData', formData); return ( @@ -209,7 +228,8 @@ const Configure = () => { )} - {formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING && + {(formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || + formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && { formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && { } + {/* {formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION || + + + + + + } */} ) diff --git a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx index e0696f1..d659833 100644 --- a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx +++ b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx @@ -19,6 +19,25 @@ export const StyledTextArea = styled(Input.TextArea)` min-height: 175px !important; `; +const StyledModal = styled(Modal)` + .ant-modal-content { + max-height: 90vh; + // height: 760px; + height: 85vh; + width: 750px; + .ant-modal-body { + padding-top: 0; + min-height: 70vh; + } + } + // .ant-modal-content { + // border-radius: 8px; + // box-shadow: 0px 4px 16px rgba(0, 0, 0, 0.1); + // background-color: #ffffff; + // padding: 24px; + // } +` + const CustomPromptButton: React.FC = ({ model_id, inference_type, caii_endpoint, use_case, setPrompt }) => { const [form] = Form.useForm(); const [showModal, setShowModal] = useState(false); @@ -40,6 +59,7 @@ const CustomPromptButton: React.FC = ({ model_id, inference_type, caii_en } }, [mutation.error, mutation.isSuccess]); + console.log('mutation', mutation); const onFinish = async () => { const custom_prompt = form.getFieldValue('custom_prompt_instructions'); try { @@ -67,7 +87,7 @@ const CustomPromptButton: React.FC = ({ model_id, inference_type, caii_en {showModal && ( - = ({ model_id, inference_type, caii_en - + ) } diff --git a/app/client/src/pages/DataGenerator/DataGenerator.tsx b/app/client/src/pages/DataGenerator/DataGenerator.tsx index 62a545b..5ebca8a 100644 --- a/app/client/src/pages/DataGenerator/DataGenerator.tsx +++ b/app/client/src/pages/DataGenerator/DataGenerator.tsx @@ -17,6 +17,7 @@ import Finish from './Finish'; import { DataGenWizardSteps, WizardStepConfig, WorkflowType } from './types'; import { WizardCtx } from './utils'; +import { useGetDatasetDetails } from '../DatasetDetails/hooks'; const { Content } = Layout; // const { Title } = Typography; @@ -98,10 +99,14 @@ const DataGenerator = () => { const [isStepValid, setIsStepValid] = useState(false); // Data passed from listing table to prepopulate form const location = useLocation(); - console.log('DatGenerator >> location?.state?.data:', location?.state?.data); + console.log('location?.state?.data:', location?.state?.data); const initialData = location?.state?.data; + + const datasetDetailsReq = location?.state?.data && useGetDatasetDetails(location?.state?.data?.generate_file_name) if (initialData?.technique) { - initialData.workflow_type = initialData?.technique === 'sft' ? WorkflowType.SUPERVISED_FINE_TUNING : + initialData.workflow_type = initialData?.technique === 'sft' ? + WorkflowType.SUPERVISED_FINE_TUNING : + initialData?.technique === 'freeform' ? WorkflowType.FREE_FORM_DATA_GENERATION : WorkflowType.CUSTOM_DATA_GENERATION; } if (Array.isArray(initialData?.doc_paths) && !isEmpty(initialData?.doc_paths) ) { @@ -111,6 +116,12 @@ const DataGenerator = () => { })); } + + if (datasetDetailsReq && datasetDetailsReq.data && + !isEmpty(datasetDetailsReq?.data?.generate_file_name)) { + initialData.example_path = initialData?.example_path; + } + if (Array.isArray(initialData?.input_paths) && !isEmpty(initialData?.input_paths) ) { initialData.doc_paths = initialData?.input_paths.map((path: string) => ({ value: path, diff --git a/app/client/src/pages/DataGenerator/Examples.tsx b/app/client/src/pages/DataGenerator/Examples.tsx index 2864ba6..3fed013 100644 --- a/app/client/src/pages/DataGenerator/Examples.tsx +++ b/app/client/src/pages/DataGenerator/Examples.tsx @@ -1,10 +1,21 @@ -import { Button, Form, Modal, Space, Table, Tooltip, Typography, Flex } from 'antd'; -import { DeleteOutlined, EditOutlined } from '@ant-design/icons'; +import first from 'lodash/first'; +import get from 'lodash/get'; +import isEmpty from 'lodash/isEmpty'; +import React, { useEffect } from 'react'; +import { Button, Form, Modal, Space, Table, Tooltip, Typography, Flex, Input, Empty, Alert } from 'antd'; +import { CloudUploadOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons'; import styled from 'styled-components'; +import { useMutation } from "@tanstack/react-query"; import { useFetchExamples } from '../../api/api'; import TooltipIcon from '../../components/TooltipIcon'; import PCModalContent from './PCModalContent'; -import { QuestionSolution } from './types'; +import { File, QuestionSolution, WorkflowType } from './types'; +import FileSelectorButton from './FileSelectorButton'; + +import { fetchFileContent } from './hooks'; +import { useState } from 'react'; +import FreeFormExampleTable from './FreeFormExampleTable'; +import { useWizardCtx } from './utils'; const { Title } = Typography; const Container = styled.div` @@ -25,20 +36,66 @@ const StyledTable = styled(Table)` cursor: pointer; } ` + +const StyledContainer = styled.div` + margin-bottom: 24px; + height: 48px; + color: rgba(0, 0, 0, 0.45); + svg { + font-size: 48px; + } + +`; + const MAX_EXAMPLES = 5; -const Examples = () => { +enum ExampleType { + FREE_FORM = 'freeform', + PROMPT_COMPLETION = 'promptcompletion' +} + +const Examples: React.FC = () => { const form = Form.useFormInstance(); - // const { setIsStepValid } = useWizardCtx(); - // const _values = Form.useWatch('examples', form); - // useEffect (() => { - // const values = form.getFieldsValue(); - // if (isEmpty(values.examples)) { - // setIsStepValid(false); - // } else if (!isEmpty(values?.examples)) { - // setIsStepValid(true); - // } - // }, [_values]); + const formData = Form.useWatch((values) => values, form); + const [exampleType, setExampleType] = useState(ExampleType.PROMPT_COMPLETION); + + const mutation = useMutation({ + mutationFn: fetchFileContent + }); + const values = form.getFieldsValue(true) + + useEffect(() => { + console.log('Examples >> form.getFieldValue(\'workflow_type\'):', form.getFieldValue('workflow_type')); + const example_path = form.getFieldValue('example_path'); + console.log('Examples >> example_path:', example_path); + + if (!isEmpty(example_path)) { + mutation.mutate({ + path: example_path + }); + } + + if (form.getFieldValue('workflow_type') === 'freeform') { + setExampleType(ExampleType.FREE_FORM); + } + + + + }, [form.getFieldValue('example_path'), form.getFieldValue('workflow_type')]); + + console.log('Examples >> mutation:', mutation); + const { setIsStepValid } = useWizardCtx(); + const _values = Form.useWatch(['examples', 'example_path'], form); + useEffect (() => { + const values = form.getFieldsValue(); + console.log('values:', values); + console.log(form.getFieldValue('example_path')); + if (isEmpty(values.examples) && isEmpty(form.getFieldValue('example_path'))) { + setIsStepValid(false); + } else { + setIsStepValid(true); + } + }, [_values, form.getFieldValue('example_path')]); const columns = [ { @@ -141,6 +198,33 @@ const Examples = () => { form.setFieldValue('examples', examples.examples) } const rowLimitReached = form.getFieldValue('examples')?.length === MAX_EXAMPLES; + const workflowType = form.getFieldValue('workflow_type'); + + const onAddFiles = (files: File[]) => { + if (!isEmpty (files)) { + const file = first(files); + console.log('Examples >> file:', file); + mutation.mutate({ + path: get(file, '_path'), + }); + const values = form.getFieldsValue(); + form.setFieldsValue({ + ...values, + example_path: get(file, '_path') + }); + setExampleType(ExampleType.FREE_FORM); + } + } + + const labelCol = { + span: 10 + }; + + console.log('exampleType:', exampleType); + console.log('data:', mutation.data); + console.log('formData', formData); + + return ( @@ -151,7 +235,26 @@ const Examples = () => { - + + {workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && + <> + + + + + + } + + {exampleType !== ExampleType.FREE_FORM && + } + + {exampleType !== ExampleType.FREE_FORM && - + } + {exampleType === ExampleType.FREE_FORM && !isEmpty(mutation.data) && + } + {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && !isEmpty(values.examples) && + } + {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && isEmpty(values.examples) && + + + + } + imageStyle={{ + height: 60, + marginBottom: 24 + }} + description={ + <> +

+ Upload a JSON file containing examples +

+

+ {'Examples should be in the format of a JSON array containing array of key & value pairs. The key should be the column name and the value should be the cell value.'} +

+ + } + > + + } + {exampleType !== ExampleType.FREE_FORM && @@ -230,7 +364,7 @@ const Examples = () => { rowClassName={() => 'hover-pointer'} rowKey={(_record, index) => `examples-table-${index}`} /> - + } ) diff --git a/app/client/src/pages/DataGenerator/FileSelectorButton.tsx b/app/client/src/pages/DataGenerator/FileSelectorButton.tsx index fd612cf..d473423 100644 --- a/app/client/src/pages/DataGenerator/FileSelectorButton.tsx +++ b/app/client/src/pages/DataGenerator/FileSelectorButton.tsx @@ -9,9 +9,10 @@ import { File, WorkflowType } from './types'; interface Props { onAddFiles: (files: File[]) => void; workflowType: WorkflowType; + label?: string; } -const FileSelectorButton: React.FC = ({ onAddFiles, workflowType }) => { +const FileSelectorButton: React.FC = ({ onAddFiles, workflowType, label }) => { const [showModal, setShowModal] = useState(false); const [selectedFiles, setSelectedFiles] = useState([]) @@ -31,7 +32,9 @@ const FileSelectorButton: React.FC = ({ onAddFiles, workflowType }) => { style={{ marginLeft: '4px' }} onClick={() => setShowModal(true)} icon={} - /> + > + {label ? label : null} + {showModal && ( { const Finish = () => { const form = Form.useFormInstance(); - const { data: genDatasetResp, loading, error: generationError, triggerPost } = useTriggerDatagen(); - const { num_questions, topics } = form.getFieldsValue(true) + const { num_questions, topics, workflow_type } = form.getFieldsValue(true); + console.log('Finish >> form:', form.getFieldsValue(true)); + console.log('Finish >> workflow_type:', workflow_type); + const { data: genDatasetResp, loading, error: generationError, triggerPost } = useTriggerDatagen(workflow_type); + const isDemo = isDemoMode(num_questions, topics, form) useEffect(() => { @@ -153,6 +158,8 @@ const Finish = () => { formValues.technique = 'sft'; } else if (formValues.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) { formValues.technique = 'custom_workflow'; + } else if (formValues.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) { + formValues.technique = 'freeform'; } // send examples as null when the array is empty if (isEmpty(formValues.examples)) { @@ -192,13 +199,30 @@ const Finish = () => { let topicTabs = []; if (!hasDocSeeds && formValues.workflow_type !== WorkflowType.CUSTOM_DATA_GENERATION && - hasTopics(genDatasetResp)) { - topicTabs = genDatasetResp?.results && Object.keys(genDatasetResp.results).map((topic, i) => ({ + hasTopics(genDatasetResp) && !isEmpty(genDatasetResp?.results)) { + console.log('Finish >> genDatasetResp:', genDatasetResp); + const values = Object.values(genDatasetResp?.results); + console.log('Finish >> values:', values); + + + topicTabs = genDatasetResp?.results && Object.keys(genDatasetResp.results).map((topic, i) => { + console.log('Finish >> topic:', topic); + console.log('Finish >> genDatasetResp:', genDatasetResp); + console.log('Finish >> genDatasetResp.results:', get(genDatasetResp, `results.${topic}`)); + console.log('Finish >> values:', values[i]); + + return ({ key: `${topic}-${i}`, label: {topic}, value: topic, - children: - })); + children: workflow_type !== WorkflowType.FREE_FORM_DATA_GENERATION ? + : + // + + + + }) + }); } const nextStepsListPreview = [ diff --git a/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx b/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx new file mode 100644 index 0000000..82bff36 --- /dev/null +++ b/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx @@ -0,0 +1,179 @@ + +import isEmpty from 'lodash/isEmpty'; +import first from 'lodash/first'; +import toString from 'lodash/toString'; +import React, { FunctionComponent, useState, useMemo, useCallback, useEffect } from 'react'; +// import { themeBalham } from '@ag-grid-community/theming'; +import { AgGridReact } from 'ag-grid-react'; +// import { Grid } from '@ag-grid-community/core'; +// import 'ag-grid-community/styles/ag-grid.min.css'; +// import 'ag-grid-community/styles/ag-theme-balham.css'; +// import 'ag-grid-community/styles/ag-theme-quartz.css'; +// // import 'ag-grid-community/styles/ag-theme-alpine.min.css'; +// import "ag-grid-community/styles/ag-grid.css"; +// import "ag-grid-community/styles/ag-theme-quartz.css"; +//import "ag-grid-community/styles/ag-grid.css"; +//import "ag-grid-community/styles/ag-theme-quartz.css"; + +// // import { AllCommunityModule, ModuleRegistry } from "ag-grid-community"; + +// // Register all Community features +// // ModuleRegistry.registerModules([AllCommunityModule]); +import { + themeAlpine, + themeBalham, + themeMaterial, + themeQuartz, +} from "ag-grid-community"; + +const themes: Record = { + quartz: themeQuartz, + material: themeMaterial, + balham: themeBalham, + alpine: themeAlpine, + }; + const theme = themeQuartz; + +import { + ModuleRegistry, + ClientSideRowModelModule, + ValidationModule, + type ColDef, + type GetRowIdFunc, + type GetRowIdParams + } from 'ag-grid-community'; + + import { themeAlpine } from 'ag-grid-community'; + + import { ModuleRegistry } from 'ag-grid-community'; +// import { RowGroupingModule } from 'ag-grid-community'; +// import { PivotModule } from 'ag-grid-community'; +// import { TreeDataModule } from 'ag-grid-community'; +// import { ClientSideRowModelModule } from 'ag-grid-community'; +// import { AllModules } from 'ag-grid-community'; +import { TextFilterModule } from 'ag-grid-community'; +import { NumberFilterModule } from 'ag-grid-community'; +import { DateFilterModule } from 'ag-grid-community'; +// import { SetFilterModule } from 'ag-grid-community'; +// import { MultiFilterModule } from 'ag-grid-community'; +// import { GroupFilterModule } from 'ag-grid-community'; +// import { CustomFilterModule } from 'ag-grid-community'; + +// Register all Community features (if needed, specify valid modules here) +ModuleRegistry.registerModules([ + // AllModules, + TextFilterModule, + NumberFilterModule, + DateFilterModule, + // SetFilterModule, + // MultiFilterModule, + // GroupFilterModule, + // CustomFilterModule, + + // ModuleRegistry, + // RowGroupingModule, + // PivotModule, + // TreeDataModule, + ClientSideRowModelModule, + ValidationModule +]); + +interface Props { + data: Record[]; +} + +const FreeFormExampleTable: FunctionComponent = ({ data }) => { + console.log('>>> FreeFormExampleTable'); + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + useEffect(() => { + console.log('>>> FreeFormExampleTable data:', data); + if (!isEmpty(data)) { + const columnNames = Object.keys(first(data)); + const columnDefs = columnNames.map((colName) => ({ + field: colName, + headerName: colName, + width: 250, + filter: true, + sortable: true, + resizable: true + })); + setColDefs(columnDefs); + setRowData(data); + } + } + , [data]); + // const [rowData, setRowData] = useState([ + // { make: "Tesla", model: "Model Y", price: 64950, electric: true }, + // { make: "Ford", model: "F-Series", price: 33850, electric: false }, + // { make: "Toyota", model: "Corolla", price: 29600, electric: false }, + // ]); + + // // Column Definitions: Defines the columns to be displayed. + // const [colDefs, setColDefs] = useState([ + // { field: "make" }, + // { field: "model" }, + // { field: "price" }, + // { field: "electric" } + // ]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + enableRowGroup: true, + enableValue: true, + + editable: true, + minWidth: 170 + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: { ticker } }: GetRowIdParams) => { + index++; + return ticker || toString(index); + }, + [] + ); + + const statusBar = useMemo( + () => ({ + statusPanels: [ + { statusPanel: "agTotalAndFilteredRowCountComponent" }, + { statusPanel: "agTotalRowCountComponent" }, + { statusPanel: "agFilteredRowCountComponent" }, + { statusPanel: "agSelectedRowCountComponent" }, + { statusPanel: "agAggregationComponent" }, + ], + }), + [] + ); + + + return ( + <> +
+ +
+ + ); +} +export default FreeFormExampleTable; \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/FreeFormTable.tsx b/app/client/src/pages/DataGenerator/FreeFormTable.tsx new file mode 100644 index 0000000..fe6ab31 --- /dev/null +++ b/app/client/src/pages/DataGenerator/FreeFormTable.tsx @@ -0,0 +1,179 @@ + +import isEmpty from 'lodash/isEmpty'; +import first from 'lodash/first'; +import toString from 'lodash/toString'; +import React, { FunctionComponent, useState, useMemo, useCallback, useEffect } from 'react'; +// import { themeBalham } from '@ag-grid-community/theming'; +import { AgGridReact } from 'ag-grid-react'; +// import { Grid } from '@ag-grid-community/core'; +// import 'ag-grid-community/styles/ag-grid.min.css'; +// import 'ag-grid-community/styles/ag-theme-balham.css'; +// import 'ag-grid-community/styles/ag-theme-quartz.css'; +// // import 'ag-grid-community/styles/ag-theme-alpine.min.css'; +// import "ag-grid-community/styles/ag-grid.css"; +// import "ag-grid-community/styles/ag-theme-quartz.css"; +//import "ag-grid-community/styles/ag-grid.css"; +//import "ag-grid-community/styles/ag-theme-quartz.css"; + +// // import { AllCommunityModule, ModuleRegistry } from "ag-grid-community"; + +// // Register all Community features +// // ModuleRegistry.registerModules([AllCommunityModule]); +import { + themeAlpine, + themeBalham, + themeMaterial, + themeQuartz, +} from "ag-grid-community"; + +const themes: Record = { + quartz: themeQuartz, + material: themeMaterial, + balham: themeBalham, + alpine: themeAlpine, + }; + const theme = themeQuartz; + +import { + ModuleRegistry, + ClientSideRowModelModule, + ValidationModule, + type ColDef, + type GetRowIdFunc, + type GetRowIdParams + } from 'ag-grid-community'; + + import { themeAlpine } from 'ag-grid-community'; + + import { ModuleRegistry } from 'ag-grid-community'; +// import { RowGroupingModule } from 'ag-grid-community'; +// import { PivotModule } from 'ag-grid-community'; +// import { TreeDataModule } from 'ag-grid-community'; +// import { ClientSideRowModelModule } from 'ag-grid-community'; +// import { AllModules } from 'ag-grid-community'; +import { TextFilterModule } from 'ag-grid-community'; +import { NumberFilterModule } from 'ag-grid-community'; +import { DateFilterModule } from 'ag-grid-community'; +// import { SetFilterModule } from 'ag-grid-community'; +// import { MultiFilterModule } from 'ag-grid-community'; +// import { GroupFilterModule } from 'ag-grid-community'; +// import { CustomFilterModule } from 'ag-grid-community'; + +// Register all Community features (if needed, specify valid modules here) +ModuleRegistry.registerModules([ + // AllModules, + TextFilterModule, + NumberFilterModule, + DateFilterModule, + // SetFilterModule, + // MultiFilterModule, + // GroupFilterModule, + // CustomFilterModule, + + // ModuleRegistry, + // RowGroupingModule, + // PivotModule, + // TreeDataModule, + ClientSideRowModelModule, + ValidationModule +]); + +interface Props { + data: Record[]; +} + +const FreeFormTable: FunctionComponent = ({ data }) => { + console.log('>>> FreeFormTable', data); + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + useEffect(() => { + console.log('>>> FreeFormExampleTable data:', data); + if (!isEmpty(data)) { + const columnNames = Object.keys(first(data)); + const columnDefs = columnNames.map((colName) => ({ + field: colName, + headerName: colName, + width: 250, + filter: true, + sortable: true, + resizable: true + })); + setColDefs(columnDefs); + setRowData(data); + } + } + , [data]); + // const [rowData, setRowData] = useState([ + // { make: "Tesla", model: "Model Y", price: 64950, electric: true }, + // { make: "Ford", model: "F-Series", price: 33850, electric: false }, + // { make: "Toyota", model: "Corolla", price: 29600, electric: false }, + // ]); + + // // Column Definitions: Defines the columns to be displayed. + // const [colDefs, setColDefs] = useState([ + // { field: "make" }, + // { field: "model" }, + // { field: "price" }, + // { field: "electric" } + // ]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + enableRowGroup: true, + enableValue: true, + + editable: true, + minWidth: 170 + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: { ticker } }: GetRowIdParams) => { + index++; + return ticker || toString(index); + }, + [] + ); + + const statusBar = useMemo( + () => ({ + statusPanels: [ + { statusPanel: "agTotalAndFilteredRowCountComponent" }, + { statusPanel: "agTotalRowCountComponent" }, + { statusPanel: "agFilteredRowCountComponent" }, + { statusPanel: "agSelectedRowCountComponent" }, + { statusPanel: "agAggregationComponent" }, + ], + }), + [] + ); + + + return ( + <> +
+ +
+ + ); +} +export default FreeFormTable; \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/Parameters.tsx b/app/client/src/pages/DataGenerator/Parameters.tsx index 7f1564e..6508bea 100644 --- a/app/client/src/pages/DataGenerator/Parameters.tsx +++ b/app/client/src/pages/DataGenerator/Parameters.tsx @@ -185,7 +185,7 @@ const Parameters = () => { - {/* {LABELS[ModelParameters.MAX_TOKENS]}} labelCol={{ span: 24 }} @@ -215,7 +215,7 @@ const Parameters = () => { /> - */} + ) diff --git a/app/client/src/pages/DataGenerator/Prompt.tsx b/app/client/src/pages/DataGenerator/Prompt.tsx index 71c654d..17ddf16 100644 --- a/app/client/src/pages/DataGenerator/Prompt.tsx +++ b/app/client/src/pages/DataGenerator/Prompt.tsx @@ -72,12 +72,14 @@ const Prompt = () => { const inference_type = form.getFieldValue('inference_type'); const doc_paths = form.getFieldValue('doc_paths'); const workflow_type = form.getFieldValue('workflow_type'); + console.log('workflow_type', workflow_type); + console.log('useCase', useCase); const input_key = form.getFieldValue('input_key'); const input_value = form.getFieldValue('input_value'); const output_key = form.getFieldValue('output_key'); const caii_endpoint = form.getFieldValue('caii_endpoint'); - const { data: defaultPrompt, loading: promptsLoading } = useFetchDefaultPrompt(useCase); + const { data: defaultPrompt, loading: promptsLoading } = useFetchDefaultPrompt(useCase, workflow_type); // Page Bootstrap requests and useEffect const { data: defaultTopics, loading: topicsLoading } = usefetchTopics(useCase); @@ -266,7 +268,8 @@ const Prompt = () => { } {isEmpty(doc_paths) && (workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || - workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && + workflow_type === WorkflowType.CUSTOM_DATA_GENERATION || + workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && { } } +export const fetchFileContent = async (params: any) => { + const resp = await fetch(`${BASE_API_URL}/json/get_content`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(params), + }); + if (resp.status !== 200) { + const error = await resp.json(); + throw new Error(error.message || error.detail); + } + const body = await resp.json(); + const content = get(body, 'data'); + return content; +} + export const listModels = async (params: any) => { const resp = await fetch(`${BASE_API_URL}/model/model_ID`, { method: 'POST', @@ -145,7 +162,7 @@ export const useGetProjectFiles = (paths: string[]) => { if (mutation.isError) { notification.error({ message: 'Error', - description: `An error occurred while fetching the prompt.\n ${mutation.error}` + description: `An error occurred while fetching the list of project files.\n ${mutation.error}` }); } return { diff --git a/app/client/src/pages/DataGenerator/types.ts b/app/client/src/pages/DataGenerator/types.ts index 38a8f25..432302f 100644 --- a/app/client/src/pages/DataGenerator/types.ts +++ b/app/client/src/pages/DataGenerator/types.ts @@ -104,7 +104,8 @@ export interface File { export enum WorkflowType { SUPERVISED_FINE_TUNING = 'supervised-fine-tuning', - CUSTOM_DATA_GENERATION = "custom" + CUSTOM_DATA_GENERATION = "custom", + FREE_FORM_DATA_GENERATION = "freeform" } export interface CustomResult { @@ -114,5 +115,6 @@ export interface CustomResult { export enum TechniqueType { SFT = 'sft', - CUSTOME_WORKFLOW = 'custom_workflow' + CUSTOME_WORKFLOW = 'custom_workflow', + FREE_FORM = 'freeform' } \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/utils.ts b/app/client/src/pages/DataGenerator/utils.ts index ab9f784..6f7e932 100644 --- a/app/client/src/pages/DataGenerator/utils.ts +++ b/app/client/src/pages/DataGenerator/utils.ts @@ -37,3 +37,35 @@ export const fromNow = time => { } return moment(time).fromNow(); }; + +export const sampleExamplesData = [ + { + "loan_amnt": 10000.00, + "term": "36 months", + "int_rate": 11.44, + "installment": 329.48, + "grade": "B", + "sub_grade": "B4", + "emp_title": "Marketing", + "emp_length": "10+ years", + "home_ownership": "RENT", + "annual_inc": 117000.00, + "verification_status": "Not Verified", + "issue_d": "Jan-2015", + "loan_status": "Fully Paid", + "purpose": "vacation", + "title": "Vacation", + "dti": 26.24, + "earliest_cr_line": "Jun-1990", + "open_acc": 16.00, + "pub_rec": 0.00, + "revol_bal": 36369.00, + "revol_util": 41.80, + "total_acc": 25.00, + "initial_list_status": "w", + "application_type": "INDIVIDUAL", + "mort_acc": 0.00, + "pub_rec_bankruptcies": 0.00, + "address": "0185 Michelle Gateway\r\nMendozaberg, OK 22690" + } +]; diff --git a/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx b/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx index 138adbb..b16ec0c 100644 --- a/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx +++ b/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx @@ -6,6 +6,7 @@ import { Col, Flex, Modal, Row, Space, Table, Tag, Typography } from 'antd'; import ExampleModal from './ExampleModal'; import { QuestionSolution } from '../DataGenerator/types'; import styled from 'styled-components'; +import FreeFormExampleTable from '../DataGenerator/FreeFormExampleTable'; const { Text } = Typography; @@ -149,6 +150,8 @@ const ConfigurationTab: React.FC = ({ dataset }) => { Examples + {dataset.technique === 'freeform' && } + {dataset.technique !== 'freeform' && = ({ dataset }) => { }) })} rowKey={(_record, index) => `summary-examples-table-${index}`} - /> + />} diff --git a/app/client/src/pages/DatasetDetails/DatasetDetailsPage.tsx b/app/client/src/pages/DatasetDetails/DatasetDetailsPage.tsx index 3ef5175..3854017 100644 --- a/app/client/src/pages/DatasetDetails/DatasetDetailsPage.tsx +++ b/app/client/src/pages/DatasetDetails/DatasetDetailsPage.tsx @@ -221,7 +221,7 @@ const DatasetDetailsPage: React.FC = () => { - Files + Context {/* {dataset?.custom_prompt} */} diff --git a/app/client/src/pages/DatasetDetails/DatasetGenerationTab.tsx b/app/client/src/pages/DatasetDetails/DatasetGenerationTab.tsx index 9341cf2..831bad3 100644 --- a/app/client/src/pages/DatasetDetails/DatasetGenerationTab.tsx +++ b/app/client/src/pages/DatasetDetails/DatasetGenerationTab.tsx @@ -7,6 +7,7 @@ import CustomGenerationTable from './CustomGenerationTable'; import DatasetGenerationTopics from './DatasetGenerationTopics'; import { CustomResult } from "../DataGenerator/types"; import { DatasetDetails, DatasetGeneration } from '../Home/types'; +import DatasetViewer from './DatasetViewer'; @@ -23,19 +24,19 @@ const Container = styled.div` const DatasetGenerationTab: React.FC = ({ dataset, datasetDetails }) => { - console.log(`DatasetGenerationTab > dataset`, dataset); - console.log(` datasetDetails`, datasetDetails); + console.log('datasetDetails', datasetDetails); + console.log('dataset', dataset); const topics = get(dataset, 'topics', []); - console.log(` topics`, topics); + const technique = get(dataset, 'technique'); const hasCustomSeeds = !Array.isArray(datasetDetails?.generation) || isEmpty(topics) || topics !== null; - console.log(` hasCustomSeeds`, hasCustomSeeds); return ( - {hasCustomSeeds && } - {!hasCustomSeeds && } + {technique === 'freeform' && } + {(technique !== 'freeform' && hasCustomSeeds) && } + {(technique !== 'freeform' && !hasCustomSeeds) && } diff --git a/app/client/src/pages/DatasetDetails/DatasetGenerationTopics.tsx b/app/client/src/pages/DatasetDetails/DatasetGenerationTopics.tsx index 3d5d529..9e33ff3 100644 --- a/app/client/src/pages/DatasetDetails/DatasetGenerationTopics.tsx +++ b/app/client/src/pages/DatasetDetails/DatasetGenerationTopics.tsx @@ -5,6 +5,7 @@ import TopicGenerationTable from "./TopicGenerationTable"; import isEmpty from "lodash/isEmpty"; import styled from "styled-components"; import { Dataset } from '../Evaluator/types'; +import FreeFormTable from '../DataGenerator/FreeFormTable'; interface Props { data: DatasetGeneration; @@ -58,16 +59,23 @@ const getTopicTree = (data: DatasetGeneration, topics: string[]) => { const DatasetGenerationTable: React.FC = ({ data, dataset }) => { + console.log('----DatasetGenerationTable', data, dataset); const topics = get(dataset, 'topics', []); + const technique = get(dataset, 'technique'); const topicTree = getTopicTree(data, topics); + const values = Object.values(topicTree); + const datasetExamples = get(dataset, 'examples', []); + let topicTabs = []; if (!isEmpty(topics)) { topicTabs = topicTree && Object.keys(topicTree).map((topic, i) => ({ key: `${topic}-${i}`, label: {topic}, value: topic, - children: + children: technique !== 'freefoem' ? + : + })); } diff --git a/app/client/src/pages/DatasetDetails/DatasetViewer.tsx b/app/client/src/pages/DatasetDetails/DatasetViewer.tsx new file mode 100644 index 0000000..11ee33f --- /dev/null +++ b/app/client/src/pages/DatasetDetails/DatasetViewer.tsx @@ -0,0 +1,46 @@ +import { FunctionComponent, useEffect } from "react"; +import { Dataset } from '../Evaluator/types'; +import { useMutation } from "@tanstack/react-query"; +import { fetchFileContent } from "../DataGenerator/hooks"; +import get from "lodash/get"; +import isEmpty from "lodash/isEmpty"; +import { Col, Row } from "antd"; +import FreeFormTable from "../DataGenerator/FreeFormTable"; + +interface Props { + dataset: Dataset; +} + + +const DatasetViewer: FunctionComponent = ({ dataset }) => { + const mutation = useMutation({ + mutationFn: fetchFileContent + }); + console.log('dataset', dataset); + console.log('mutation', mutation); + + useEffect(() => { + const generate_file_name = get(dataset, 'generate_file_name'); + if (!isEmpty(generate_file_name)) { + mutation.mutate({ + path: generate_file_name + }); + } + }, [dataset]); + + + return ( + + +
+ {mutation.isLoading &&

Loading...

} + {mutation.isError &&

Error: {mutation.error}

} + {mutation.isSuccess && ( + + )} +
+ + + ); +} +export default DatasetViewer; \ No newline at end of file diff --git a/app/client/src/pages/DatasetDetails/ExamplesSection.tsx b/app/client/src/pages/DatasetDetails/ExamplesSection.tsx index aaf5d52..d93662d 100644 --- a/app/client/src/pages/DatasetDetails/ExamplesSection.tsx +++ b/app/client/src/pages/DatasetDetails/ExamplesSection.tsx @@ -8,6 +8,7 @@ import { Dataset } from "../../../pages/Evaluator/types"; import PCModalContent from "../../../pages/DataGenerator/PCModalContent"; import ExampleModal from "./ExampleModal"; +import FreeFormExampleTable from "../DataGenerator/FreeFormExampleTable"; const { Text, Title } = Typography; const Panel = Collapse.Panel; @@ -74,6 +75,8 @@ export type DatasetDetailProps = { } const ExamplesSection= ({ datasetDetails }: DatasetDetailProps) => { + console.log('ExamplesSection >> datasetDetails', datasetDetails); + const { technique } = datasetDetails; const exampleCols = [ { @@ -99,6 +102,11 @@ const ExamplesSection= ({ datasetDetails }: DatasetDetailProps) => { style={{ padding: 0 }} > + {technique === 'freeform' ? ( + + ) : { }) })} rowKey={(_record, index) => `summary-examples-table-${index}`} - /> - - {/* Model Parameters - ({ - label: MODEL_PARAMETER_LABELS[modelParameterKey as ModelParameters], - children: datasetDetails.model_parameters[modelParameterKey as ModelParameters], - })) - : []}> - - {(datasetDetails.schema && datasetDetails.use_case === Usecases.TEXT2SQL) && ( -
- {'DB Schema'} - - - -
- )} */} - + />} diff --git a/app/client/src/pages/Evaluator/EvaluateDataset.tsx b/app/client/src/pages/Evaluator/EvaluateDataset.tsx index 6f6e8ae..b0a9294 100644 --- a/app/client/src/pages/Evaluator/EvaluateDataset.tsx +++ b/app/client/src/pages/Evaluator/EvaluateDataset.tsx @@ -56,6 +56,7 @@ const StyledCard = styled(Card)` const EvaluateDataset: React.FC = ({ form, loading, modelsMap, dataset, examples, viewType, onEvaluate, evaluate }) => { + console.log('EvaluateDataset', dataset, examples); const inference_type = get(dataset, 'inference_type', ''); const [models, setModels] = useState(get(modelsMap, inference_type, [])); const selectedInferenceType = Form.useWatch('inference_type', form); diff --git a/app/client/src/pages/Evaluator/GeneratedEvaluationModal.tsx b/app/client/src/pages/Evaluator/GeneratedEvaluationModal.tsx index 3df4f4e..057fcc0 100644 --- a/app/client/src/pages/Evaluator/GeneratedEvaluationModal.tsx +++ b/app/client/src/pages/Evaluator/GeneratedEvaluationModal.tsx @@ -2,14 +2,12 @@ import get from 'lodash/get'; import isString from 'lodash/isString'; import React from 'react'; import { EvaluatedPair } from "./types"; -import { Badge, Button, Flex, Layout, Modal, Tooltip } from 'antd'; +import { Badge, Button, Flex, Modal, Tooltip } from 'antd'; import { QuestionCircleOutlined } from '@ant-design/icons'; import styled from 'styled-components'; import Markdown from '../../components/Markdown'; import { getColorCode } from './util'; -const { Content } = Layout; - interface Props { evaluatedPair: EvaluatedPair; onClose: () => void; diff --git a/app/client/src/pages/Home/DatasetsTab.tsx b/app/client/src/pages/Home/DatasetsTab.tsx index 84b54ea..f67c9c5 100644 --- a/app/client/src/pages/Home/DatasetsTab.tsx +++ b/app/client/src/pages/Home/DatasetsTab.tsx @@ -86,7 +86,7 @@ const DatasetsTab: React.FC = () => { } }, [exportResult, notificationInstance]) - const onSearch: SearchProps['onSearch'] = (value, _e, info) => { + const onSearch: SearchProps['onSearch'] = (value: any) => { throttle((value: string) => setSearchQuery(value), 500)(value); } diff --git a/app/client/src/pages/Home/EvaluateButton.tsx b/app/client/src/pages/Home/EvaluateButton.tsx index 4cb5d32..10ea82f 100644 --- a/app/client/src/pages/Home/EvaluateButton.tsx +++ b/app/client/src/pages/Home/EvaluateButton.tsx @@ -8,7 +8,6 @@ import { isEmpty } from "lodash"; import { Dataset } from "../Evaluator/types"; import { Pages } from "../../types"; -const { Option } = Select; const EvaluateButton: React.FC = () => { const [form] = Form.useForm(); diff --git a/app/client/src/pages/Home/EvaluationsTab.tsx b/app/client/src/pages/Home/EvaluationsTab.tsx index 62e8510..348e595 100644 --- a/app/client/src/pages/Home/EvaluationsTab.tsx +++ b/app/client/src/pages/Home/EvaluationsTab.tsx @@ -65,7 +65,7 @@ const EvaluationsTab: React.FC = () => { } }, [isError]); - const onSearch: SearchProps['onSearch'] = (value, _e, info) => { + const onSearch: SearchProps['onSearch'] = (value: any) => { throttle((value: string) => setSearchQuery(value), 500)(value); } diff --git a/build/shell_scripts/build_client.sh b/build/shell_scripts/build_client.sh index d70a30c..2a0e6bb 100644 --- a/build/shell_scripts/build_client.sh +++ b/build/shell_scripts/build_client.sh @@ -16,6 +16,7 @@ fi # Activate virtual environment - using relative path source .venv/bin/activate +export NODE_OPTIONS=--max-old-space-size=16384 # Build frontend cd "$CLIENT_DIR" rm -rf node_modules/