diff --git a/extensions/model-extension/src/index.test.ts b/extensions/model-extension/src/index.test.ts index 5b126d4cc5..3f804b6d6f 100644 --- a/extensions/model-extension/src/index.test.ts +++ b/extensions/model-extension/src/index.test.ts @@ -784,5 +784,63 @@ describe('JanModelExtension', () => { expect.anything() ) }) + + it('should handle model with valid chat_template', async () => { + executeMock.mockResolvedValue('{prompt}') + ;(gguf as jest.Mock).mockResolvedValue({ + metadata: {}, + }) + // @ts-ignore + global.NODE = 'node' + // @ts-ignore + global.DEFAULT_MODEL = { + parameters: { stop: [] }, + settings: { + prompt_template: '<|im-start|>{prompt}<|im-end|>', + }, + } + + const result = await sut.retrieveGGUFMetadata({}) + + expect(result).toEqual({ + parameters: { + stop: [], + }, + settings: { + ctx_len: 4096, + ngl: 33, + prompt_template: '{prompt}', + }, + }) + }) + + it('should handle model without chat_template', async () => { + executeMock.mockRejectedValue({}) + ;(gguf as jest.Mock).mockResolvedValue({ + metadata: {}, + }) + // @ts-ignore + global.NODE = 'node' + // @ts-ignore + global.DEFAULT_MODEL = { + parameters: { stop: [] }, + settings: { + prompt_template: '<|im-start|>{prompt}<|im-end|>', + }, + } + + const result = await sut.retrieveGGUFMetadata({}) + + expect(result).toEqual({ + parameters: { + stop: [], + }, + settings: { + ctx_len: 4096, + ngl: 33, + prompt_template: '<|im-start|>{prompt}<|im-end|>', + }, + }) + }) }) }) diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index 20d23b7479..6d26d576c7 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -577,7 +577,7 @@ export default class JanModelExtension extends ModelExtension { dirName, binaryFileName, ]) - ) + ).catch(() => undefined) const updatedModel = await this.retrieveGGUFMetadata(metadata) @@ -877,8 +877,13 @@ export default class JanModelExtension extends ModelExtension { * @returns */ async retrieveGGUFMetadata(metadata: any): Promise> { - const template = await executeOnMain(NODE, 'renderJinjaTemplate', metadata) const defaultModel = DEFAULT_MODEL as Model + var template = await executeOnMain( + NODE, + 'renderJinjaTemplate', + metadata + ).catch(() => undefined) + const eos_id = metadata['tokenizer.ggml.eos_token_id'] const architecture = metadata['general.architecture'] diff --git a/extensions/model-extension/src/node/index.ts b/extensions/model-extension/src/node/index.ts index 6323d7f97c..2acf6ec4a8 100644 --- a/extensions/model-extension/src/node/index.ts +++ b/extensions/model-extension/src/node/index.ts @@ -16,12 +16,7 @@ export const retrieveGGUFMetadata = async (ggufPath: string) => { // Parse metadata and tensor info const { metadata } = ggufMetadata(buffer.buffer) - // Parse jinja template - const renderedTemplate = renderJinjaTemplate(metadata) - return { - ...metadata, - parsed_chat_template: renderedTemplate, - } + return metadata } catch (e) { console.log('[MODEL_EXT]', e) }