From 95632788e40c9f5ca4e0ee8b0e9760babd2b690c Mon Sep 17 00:00:00 2001 From: NamH Date: Wed, 17 Apr 2024 19:14:51 +0700 Subject: [PATCH] chore: default context length to 2048 (#2746) --- core/src/browser/extensions/model.ts | 1 + .../resources/default_settings.json | 2 +- .../resources/default-model.json | 2 +- extensions/model-extension/src/index.ts | 2 +- web/helpers/atoms/Model.atom.ts | 2 + web/hooks/useModels.ts | 23 +++++++-- .../ModelDownloadRow/index.tsx | 50 ++++++++----------- 7 files changed, 48 insertions(+), 34 deletions(-) diff --git a/core/src/browser/extensions/model.ts b/core/src/browser/extensions/model.ts index 30480bf5d4..5b3089403f 100644 --- a/core/src/browser/extensions/model.ts +++ b/core/src/browser/extensions/model.ts @@ -32,4 +32,5 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter abstract importModels(models: ImportingModel[], optionType: OptionType): Promise abstract updateModelInfo(modelInfo: Partial): Promise abstract fetchHuggingFaceRepoData(repoId: string): Promise + abstract getDefaultModel(): Promise } diff --git a/extensions/inference-nitro-extension/resources/default_settings.json b/extensions/inference-nitro-extension/resources/default_settings.json index 39f0880b0d..09d014a12c 100644 --- a/extensions/inference-nitro-extension/resources/default_settings.json +++ b/extensions/inference-nitro-extension/resources/default_settings.json @@ -27,7 +27,7 @@ "min": 0, "max": 4096, "step": 128, - "value": 4096 + "value": 2048 } } ] diff --git a/extensions/model-extension/resources/default-model.json b/extensions/model-extension/resources/default-model.json index f887a2b7a3..f2e15d2c9b 100644 --- a/extensions/model-extension/resources/default-model.json +++ b/extensions/model-extension/resources/default-model.json @@ -13,7 +13,7 @@ "created": 0, "description": "User self import model", "settings": { - "ctx_len": 4096, + "ctx_len": 2048, "embedding": false, "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:", "llama_model_path": "N/A" diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 31e20429ea..c91f75ae59 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -551,7 +551,7 @@ export default class JanModelExtension extends ModelExtension { return model } - private async getDefaultModel(): Promise { + override async getDefaultModel(): Promise { const defaultModel = DEFAULT_MODEL as Model return defaultModel } diff --git a/web/helpers/atoms/Model.atom.ts b/web/helpers/atoms/Model.atom.ts index da6dc59182..b50293df43 100644 --- a/web/helpers/atoms/Model.atom.ts +++ b/web/helpers/atoms/Model.atom.ts @@ -46,6 +46,8 @@ export const removeDownloadedModelAtom = atom( export const configuredModelsAtom = atom([]) +export const defaultModelAtom = atom(undefined) + /// TODO: move this part to another atom // store the paths of the models that are being imported export const importingModelsAtom = atom([]) diff --git a/web/hooks/useModels.ts b/web/hooks/useModels.ts index b2aa0b518a..5a6f13e035 100644 --- a/web/hooks/useModels.ts +++ b/web/hooks/useModels.ts @@ -13,25 +13,37 @@ import { useSetAtom } from 'jotai' import { extensionManager } from '@/extension' import { configuredModelsAtom, + defaultModelAtom, downloadedModelsAtom, } from '@/helpers/atoms/Model.atom' const useModels = () => { const setDownloadedModels = useSetAtom(downloadedModelsAtom) const setConfiguredModels = useSetAtom(configuredModelsAtom) + const setDefaultModel = useSetAtom(defaultModelAtom) const getData = useCallback(() => { const getDownloadedModels = async () => { const models = await getLocalDownloadedModels() setDownloadedModels(models) } + const getConfiguredModels = async () => { const models = await getLocalConfiguredModels() setConfiguredModels(models) } - getDownloadedModels() - getConfiguredModels() - }, [setDownloadedModels, setConfiguredModels]) + + const getDefaultModel = async () => { + const defaultModel = await getLocalDefaultModel() + setDefaultModel(defaultModel) + } + + Promise.all([ + getDownloadedModels(), + getConfiguredModels(), + getDefaultModel(), + ]) + }, [setDownloadedModels, setConfiguredModels, setDefaultModel]) useEffect(() => { // Try get data on mount @@ -46,6 +58,11 @@ const useModels = () => { }, [getData]) } +const getLocalDefaultModel = async (): Promise => + extensionManager + .get(ExtensionTypeEnum.Model) + ?.getDefaultModel() + const getLocalConfiguredModels = async (): Promise => extensionManager .get(ExtensionTypeEnum.Model) diff --git a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx index 85ab76a416..a215efaab9 100644 --- a/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx +++ b/web/screens/Settings/HuggingFaceRepoDetailModal/ModelDownloadRow/index.tsx @@ -3,7 +3,6 @@ import { useCallback, useMemo } from 'react' import { DownloadState, HuggingFaceRepoData, - InferenceEngine, Model, Quantization, } from '@janhq/core' @@ -23,7 +22,10 @@ import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { assistantsAtom } from '@/helpers/atoms/Assistant.atom' import { importHuggingFaceModelStageAtom } from '@/helpers/atoms/HuggingFace.atom' -import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom' +import { + defaultModelAtom, + downloadedModelsAtom, +} from '@/helpers/atoms/Model.atom' type Props = { index: number @@ -52,15 +54,15 @@ const ModelDownloadRow: React.FC = ({ const isDownloaded = downloadedModels.find((md) => md.id === fileName) != null const setHfImportingStage = useSetAtom(importHuggingFaceModelStageAtom) + const defaultModel = useAtomValue(defaultModelAtom) const model = useMemo(() => { - const promptData: string = - (repoData.cardData['prompt_template'] as string) ?? - '{system_message}\n### Instruction: {prompt}\n### Response:' + if (!defaultModel) { + return undefined + } + const model: Model = { - object: 'model', - version: '1.0', - format: 'gguf', + ...defaultModel, sources: [ { url: downloadUrl, @@ -70,38 +72,26 @@ const ModelDownloadRow: React.FC = ({ id: fileName, name: fileName, created: Date.now(), - description: 'User self import model', - settings: { - ctx_len: 4096, - embedding: false, - prompt_template: promptData, - llama_model_path: 'N/A', - }, - parameters: { - temperature: 0.7, - top_p: 0.95, - stream: true, - max_tokens: 2048, - stop: [''], - frequency_penalty: 0.7, - presence_penalty: 0, - }, metadata: { author: 'User', tags: repoData.tags, size: fileSize, }, - engine: InferenceEngine.nitro, } + console.log('NamH model: ', JSON.stringify(model)) return model - }, [fileName, fileSize, repoData, downloadUrl]) + }, [fileName, fileSize, repoData, downloadUrl, defaultModel]) const onAbortDownloadClick = useCallback(() => { - abortModelDownload(model) + if (model) { + abortModelDownload(model) + } }, [model, abortModelDownload]) const onDownloadClick = useCallback(async () => { - downloadModel(model) + if (model) { + downloadModel(model) + } }, [model, downloadModel]) const onUseModelClick = useCallback(async () => { @@ -120,6 +110,10 @@ const ModelDownloadRow: React.FC = ({ setHfImportingStage, ]) + if (!model) { + return null + } + return (