Skip to content

Commit

Permalink
v4.4.1 (FlowiseAI#294)
Browse files Browse the repository at this point in the history
* move file

* perf: dataset file manage

* v441 description

* fix: qa csv update file

* feat: rename file

* frontend show system-version
  • Loading branch information
c121914yu committed Sep 13, 2023
1 parent be3b680 commit a19afca
Show file tree
Hide file tree
Showing 53 changed files with 570 additions and 301 deletions.
4 changes: 2 additions & 2 deletions client/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "fastgpt",
"version": "3.7",
"private": true,
"version": "4.2.1",
"private": false,
"scripts": {
"dev": "next dev",
"build": "next build",
Expand Down
4 changes: 4 additions & 0 deletions client/public/locales/en/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Confirm": "Yes",
"Create New": "Create",
"Dataset": "Dataset",
"Export": "Export",
"Folder": "Folder",
"Move": "Move",
"Name": "Name",
Expand Down Expand Up @@ -215,6 +216,9 @@
"Response Detail": "Detail",
"Response Detail tips": "Whether detailed data such as references and full context need to be returned"
},
"system": {
"Help Document": "Document"
},
"user": {
"Account": "Account",
"Amount of earnings": "Earnings",
Expand Down
4 changes: 4 additions & 0 deletions client/public/locales/zh/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Confirm": "确认",
"Create New": "新建",
"Dataset": "知识库",
"Export": "导出",
"Folder": "文件夹",
"Move": "移动",
"Name": "名称",
Expand Down Expand Up @@ -215,6 +216,9 @@
"Response Detail": "返回详情",
"Response Detail tips": "是否需要返回引用、完整上下文等详细数据"
},
"system": {
"Help Document": "帮助文档"
},
"user": {
"Account": "账号",
"Amount of earnings": "收益(¥)",
Expand Down
8 changes: 8 additions & 0 deletions client/src/api/core/dataset/file.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { RequestPaging } from '../../../types/index';

export type GetFileListProps = RequestPaging & {
kbId: string;
searchText: string;
};

export type UpdateFileProps = { id: string; name?: string; datasetUsed?: boolean };
15 changes: 15 additions & 0 deletions client/src/api/core/dataset/file.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { GET, POST, PUT, DELETE } from '@/api/request';
import type { FileInfo, KbFileItemType } from '@/types/plugin';

import type { GetFileListProps, UpdateFileProps } from './file.d';

export const getDatasetFiles = (data: GetFileListProps) =>
POST<KbFileItemType[]>(`/core/dataset/file/list`, data);
export const delDatasetFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/core/dataset/file/delById`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/core/dataset/file/detail`, { fileId });
export const delDatasetEmptyFiles = (kbId: string) =>
DELETE(`/core/dataset/file/delEmptyFiles`, { kbId });

export const updateDatasetFile = (data: UpdateFileProps) => PUT(`/core/dataset/file/update`, data);
28 changes: 3 additions & 25 deletions client/src/api/plugins/kb.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
import { GET, POST, PUT, DELETE } from '../request';
import type {
DatasetItemType,
FileInfo,
KbFileItemType,
KbItemType,
KbListItemType,
KbPathItemType
} from '@/types/plugin';
import type { DatasetItemType, KbItemType, KbListItemType, KbPathItemType } from '@/types/plugin';
import { TrainingModeEnum } from '@/constants/plugin';
import {
Props as PushDataProps,
Expand All @@ -17,12 +10,7 @@ import {
Response as SearchTestResponse
} from '@/pages/api/openapi/kb/searchTest';
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type {
KbUpdateParams,
CreateKbParams,
GetKbDataListProps,
GetFileListProps
} from '../request/kb';
import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
import { QuoteItemType } from '@/types/chat';
import { KbTypeEnum } from '@/constants/kb';

Expand All @@ -42,24 +30,14 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat

export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);

/* kb file */
export const getKbFiles = (data: GetFileListProps) =>
POST<KbFileItemType[]>(`/plugins/kb/file/list`, data);
export const deleteKbFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/plugins/kb/file/delFileByFileId`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/plugins/kb/file/getFileInfo`, { fileId });
export const delEmptyFiles = (kbId: string) =>
DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId });

/* kb data */
export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data);

/**
* 获取导出数据(不分页)
*/
export const getExportDataList = (data: { kbId: string; fileId: string }) =>
export const getExportDataList = (data: { kbId: string }) =>
GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
timeout: 600000
});
Expand Down
5 changes: 0 additions & 5 deletions client/src/api/request/kb.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ export type CreateKbParams = {
type: `${KbTypeEnum}`;
};

export type GetFileListProps = RequestPaging & {
kbId: string;
searchText: string;
};

export type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
Expand Down
18 changes: 18 additions & 0 deletions client/src/api/support/file.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { GET, POST } from '../request';

import { AxiosProgressEvent } from 'axios';

export const uploadImg = (base64Img: string) => POST<string>('/system/uploadImage', { base64Img });

export const postUploadFiles = (
data: FormData,
onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) =>
POST<string[]>('/support/file/upload', data, {
onUploadProgress,
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
}
});

export const getFileViewUrl = (fileId: string) => GET<string>('/support/file/readUrl', { fileId });
16 changes: 0 additions & 16 deletions client/src/api/system.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,4 @@
import { GET, POST, PUT } from './request';
import type { InitDateResponse } from '@/pages/api/system/getInitData';
import { AxiosProgressEvent } from 'axios';

export const getInitData = () => GET<InitDateResponse>('/system/getInitData');

export const uploadImg = (base64Img: string) => POST<string>('/system/uploadImage', { base64Img });

export const postUploadFiles = (
data: FormData,
onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) =>
POST<string[]>('/plugins/file/upload', data, {
onUploadProgress,
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
}
});

export const getFileViewUrl = (fileId: string) => GET<string>('/plugins/file/readUrl', { fileId });
15 changes: 1 addition & 14 deletions client/src/components/Layout/navbar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -167,20 +167,7 @@ const Navbar = ({ unread }: { unread: number }) => {
</Link>
</Box>
)}
{feConfigs?.show_doc && (
<MyTooltip label={t('home.Docs')} placement={'right-end'}>
<Box
{...itemStyles}
mb={0}
color={'#9096a5'}
onClick={() => {
window.open(`https://doc.fastgpt.run/docs/intro`);
}}
>
<MyIcon name={'courseLight'} width={'26px'} height={'26px'} />
</Box>
</MyTooltip>
)}

<Language {...itemStyles} />
{feConfigs?.show_git && (
<MyTooltip label={`Git Star: ${gitStar}`} placement={'right-end'}>
Expand Down
2 changes: 1 addition & 1 deletion client/src/components/MyInput/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ interface Props extends InputProps {
const MyInput = ({ leftIcon, ...props }: Props) => {
return (
<Flex position={'relative'} alignItems={'center'}>
<Input w={'100%'} pl={leftIcon ? '30px' : 3} {...props} />
<Input w={'100%'} pl={leftIcon ? '30px !important' : 3} {...props} />
{leftIcon && (
<Flex
alignItems={'center'}
Expand Down
2 changes: 1 addition & 1 deletion client/src/constants/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ export const TrainingTypeMap = {
[TrainingModeEnum.index]: 'index'
};

export const PgTrainingTableName = 'modeldata';
export const PgDatasetTableName = 'modeldata';
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import React, { useCallback, useRef } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';

export const useEditInfo = ({
export const useEditTitle = ({
title,
placeholder = ''
}: {
Expand Down
55 changes: 42 additions & 13 deletions client/src/pages/account/components/Info.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic';
import { useSelectFile } from '@/hooks/useSelectFile';
import { compressImg } from '@/utils/file';
import { feConfigs } from '@/store/static';
import { feConfigs, systemVersion } from '@/store/static';
import { useTranslation } from 'next-i18next';
import { timezoneList } from '@/utils/user';
import Loading from '@/components/Loading';
Expand Down Expand Up @@ -172,20 +172,49 @@ const UserInfo = () => {
{t('user.Change')}
</Button>
</Flex>
{feConfigs?.show_userDetail && (
<Box mt={6} whiteSpace={'nowrap'} w={['85%', '300px']}>
<Flex alignItems={'center'}>
<Box flex={'0 0 80px'}>{t('user.Balance')}:&nbsp;</Box>
<Box flex={1}>
<strong>{userInfo?.balance.toFixed(3)}</strong>
</Box>
<Button size={['sm', 'md']} ml={5} onClick={onOpenPayModal}>
{t('user.Pay')}
</Button>
</Flex>
</Box>
)}
{feConfigs?.show_doc && (
<>
<Flex
mt={4}
w={['85%', '300px']}
py={3}
px={6}
border={theme.borders.sm}
borderWidth={'1.5px'}
borderRadius={'md'}
alignItems={'center'}
cursor={'pointer'}
userSelect={'none'}
onClick={() => {
window.open(`https://doc.fastgpt.run/docs/intro`);
}}
>
<MyIcon name={'courseLight'} w={'18px'} />
<Box ml={2} flex={1}>
{t('system.Help Document')}
</Box>
<Box w={'8px'} h={'8px'} borderRadius={'50%'} bg={'#67c13b'} />
<Box fontSize={'md'} ml={2}>
V{systemVersion}
</Box>
</Flex>
</>
)}
{feConfigs?.show_userDetail && (
<>
<Box mt={6} whiteSpace={'nowrap'} w={['85%', '300px']}>
<Flex alignItems={'center'}>
<Box flex={'0 0 80px'}>{t('user.Balance')}:&nbsp;</Box>
<Box flex={1}>
<strong>{userInfo?.balance.toFixed(3)}</strong>
</Box>
<Button size={['sm', 'md']} ml={5} onClick={onOpenPayModal}>
{t('user.Pay')}
</Button>
</Flex>
</Box>

<Divider my={3} />

<MyTooltip label={'点击配置账号'}>
Expand Down
6 changes: 3 additions & 3 deletions client/src/pages/api/admin/initv43.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@/constants/plugin';

export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
Expand All @@ -12,7 +12,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { rowCount } = await PgClient.query(`SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = '${PgTrainingTableName}'
AND table_name = '${PgDatasetTableName}'
AND column_name = 'file_id'`);

if (rowCount > 0) {
Expand All @@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)

jsonRes(res, {
data: await PgClient.query(
`ALTER TABLE ${PgTrainingTableName} ADD COLUMN file_id VARCHAR(100)`
`ALTER TABLE ${PgDatasetTableName} ADD COLUMN file_id VARCHAR(100)`
)
});
} catch (error) {
Expand Down
4 changes: 2 additions & 2 deletions client/src/pages/api/admin/initv44.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum } from '@/constants/kb';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@/constants/plugin';

export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
Expand All @@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
);

const response = await PgClient.update(PgTrainingTableName, {
const response = await PgClient.update(PgDatasetTableName, {
where: [['file_id', 'undefined']],
values: [{ key: 'file_id', value: '' }]
});
Expand Down
35 changes: 35 additions & 0 deletions client/src/pages/api/admin/initv441.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import mongoose from 'mongoose';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';

export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });

const data = await mongoose.connection.db
.collection('dataset.files')
.updateMany({}, { $set: { 'metadata.datasetUsed': true } });

// update pg data
const pg = await PgClient.query(`UPDATE ${PgDatasetTableName}
SET file_id = ''
WHERE (file_id = 'undefined' OR LENGTH(file_id) < 20) AND file_id != '';`);

jsonRes(res, {
data: {
data,
pg
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
Loading

0 comments on commit a19afca

Please sign in to comment.