From 6a5685995f851dd17a88e431b651d2fed732b902 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Wed, 7 Aug 2024 02:39:41 -0700 Subject: [PATCH 1/3] OpenAI: update models (cherry picked from commit 5f5efe61330f83cd91c979497704b86317c11419) --- src/modules/llms/server/openai/models.data.ts | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/modules/llms/server/openai/models.data.ts b/src/modules/llms/server/openai/models.data.ts index d108b789c..fd47d991e 100644 --- a/src/modules/llms/server/openai/models.data.ts +++ b/src/modules/llms/server/openai/models.data.ts @@ -22,7 +22,7 @@ const _knownOpenAIChatModels: ManualMappings = [ trainingDataCutoff: 'Oct 2023', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], pricing: { chatIn: 0.15, chatOut: 0.60 }, - benchmark: { cbaMmlu: 82.0 }, + benchmark: { cbaElo: 1277, cbaMmlu: 82.0 }, }, { idPrefix: 'gpt-4o-mini', @@ -36,10 +36,10 @@ const _knownOpenAIChatModels: ManualMappings = [ trainingDataCutoff: 'Oct 2023', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], pricing: { chatIn: 0.15, chatOut: 0.60 }, - benchmark: { cbaMmlu: 82.0 }, + benchmark: { cbaElo: 1277, cbaMmlu: 82.0 }, }, - // GPT-4o -> 2024-05-13 + // GPT-4o -> 2024-05-13 (will be update to 2024-08-06 in the future (3 weeks notice)) { idPrefix: 'gpt-4o', label: 'GPT-4o', @@ -52,7 +52,19 @@ const _knownOpenAIChatModels: ManualMappings = [ trainingDataCutoff: 'Oct 2023', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], pricing: { chatIn: 5, chatOut: 15 }, - benchmark: { cbaElo: 1287 }, + benchmark: { cbaElo: 1286 }, + }, + { + isLatest: true, + idPrefix: 'gpt-4o-2024-08-06', + label: 'GPT-4o (2024-08-06)', + description: 'Latest snapshot that supports Structured Outputs', + contextWindow: 128000, + maxCompletionTokens: 16384, + trainingDataCutoff: 'Oct 2023', + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], + pricing: { chatIn: 2.5, chatOut: 10 }, + benchmark: { cbaElo: 1286 + 1 }, }, { idPrefix: 'gpt-4o-2024-05-13', @@ -63,7 +75,8 @@ const _knownOpenAIChatModels: ManualMappings = [ trainingDataCutoff: 'Oct 2023', interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], pricing: { chatIn: 5, chatOut: 15 }, - benchmark: { cbaElo: 1287 }, + benchmark: { cbaElo: 1286 }, + hidden: true, }, // GPT4 Turbo with Vision -> 2024-04-09 @@ -112,8 +125,7 @@ const _knownOpenAIChatModels: ManualMappings = [ { idPrefix: 'gpt-4-0125-preview', // GPT-4 Turbo preview model label: 'GPT-4 Turbo (0125)', - description: 'GPT-4 Turbo preview model intended to reduce cases of "laziness" where the model doesn\'t complete a task. Returns a maximum of 4,096 output tokens.', - isPreview: true, + description: 'GPT-4 Turbo preview model intended to reduce cases of "laziness" where the model doesn\'t complete a task.', contextWindow: 128000, maxCompletionTokens: 4096, trainingDataCutoff: 'Dec 2023', @@ -125,12 +137,11 @@ const _knownOpenAIChatModels: ManualMappings = [ { idPrefix: 'gpt-4-1106-preview', // GPT-4 Turbo preview model label: 'GPT-4 Turbo (1106)', - description: 'GPT-4 Turbo preview model featuring improved instruction following, JSON mode, reproducible outputs, parallel function calling, and more. Returns a maximum of 4,096 output tokens.', - isPreview: true, + description: 'GPT-4 Turbo preview model featuring improved instruction following, JSON mode, reproducible outputs, parallel function calling, and more.', contextWindow: 128000, maxCompletionTokens: 4096, trainingDataCutoff: 'Apr 2023', - interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], + interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Json], pricing: { chatIn: 10, chatOut: 30 }, benchmark: { cbaElo: 1251 }, hidden: true, @@ -234,6 +245,7 @@ const _knownOpenAIChatModels: ManualMappings = [ interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Fn], pricing: { chatIn: 30, chatOut: 60 }, benchmark: { cbaElo: 1161 }, + isLegacy: true, }, @@ -249,12 +261,11 @@ const _knownOpenAIChatModels: ManualMappings = [ hidden: true, }, - - // 3.5-Turbo-16k's + // 3.5-Turbo's (16ks) { idPrefix: 'gpt-3.5-turbo-0125', label: '3.5-Turbo (0125)', - description: 'The latest GPT-3.5 Turbo model with higher accuracy at responding in requested formats and a fix for a bug which caused a text encoding issue for non-English language function calls. Returns a maximum of 4,096 output tokens.', + description: 'The latest GPT-3.5 Turbo model with higher accuracy at responding in requested formats and a fix for a bug which caused a text encoding issue for non-English language function calls.', contextWindow: 16385, maxCompletionTokens: 4096, trainingDataCutoff: 'Sep 2021', @@ -265,7 +276,7 @@ const _knownOpenAIChatModels: ManualMappings = [ { idPrefix: 'gpt-3.5-turbo-1106', label: '3.5-Turbo (1106)', - description: 'The latest GPT-3.5 Turbo model with improved instruction following, JSON mode, reproducible outputs, parallel function calling, and more.', + description: 'GPT-3.5 Turbo model with improved instruction following, JSON mode, reproducible outputs, parallel function calling, and more.', contextWindow: 16385, maxCompletionTokens: 4096, trainingDataCutoff: 'Sep 2021', From 4823e9778344628f7e9c17abf91590f8f3f65e7b Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Wed, 7 Aug 2024 02:46:33 -0700 Subject: [PATCH 2/3] Mapping doc, for the future. (cherry picked from commit a416cafc4e13dd1aa4489f868a65dc37f517416f) --- src/modules/llms/server/openai/models.cba.ts | 84 ++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/modules/llms/server/openai/models.cba.ts diff --git a/src/modules/llms/server/openai/models.cba.ts b/src/modules/llms/server/openai/models.cba.ts new file mode 100644 index 000000000..7dbf7ba15 --- /dev/null +++ b/src/modules/llms/server/openai/models.cba.ts @@ -0,0 +1,84 @@ +// here for reference only - for future mapping of CBA scores to the model IDs +// const modelIdToPrefixMap: { [key: string]: string } = { +// // Anthropic models +// 'Claude 3.5 Sonnet': 'claude-3-5-sonnet-20240620', +// 'Claude 3 Opus': 'claude-3-opus-20240229', +// 'Claude 3 Sonnet': 'claude-3-sonnet-20240229', +// 'Claude 3 Haiku': 'claude-3-haiku-20240307', +// 'Claude-2.1': 'claude-2.1', +// 'Claude-2.0': 'claude-2.0', +// 'Claude-1': '', // No exact match +// 'Claude-Instant-1': 'claude-instant-1.2', // Closest match +// +// // Gemini models +// 'Gemini-1.5-Pro-Exp-0801': 'models/gemini-1.5-pro-latest', // Closest match +// 'Gemini Advanced App (2024-05-14)': '', // No exact match +// 'Gemini-1.5-Pro-001': 'models/gemini-1.5-pro-001', +// 'Gemini-1.5-Pro-Preview-0409': 'models/gemini-1.5-pro-latest', // Closest match +// 'Gemini-1.5-Flash-001': 'models/gemini-1.5-flash-001', +// 'Gemini App (2024-01-24)': '', // No exact match +// 'Gemini-1.0-Pro-001': 'models/gemini-1.0-pro-001', +// 'Gemini Pro': 'models/gemini-pro', +// +// // OpenAI models (from the previous file) +// 'GPT-4o-2024-05-13': 'gpt-4o-2024-05-13', +// 'GPT-4o-mini-2024-07-18': 'gpt-4o-mini-2024-07-18', +// 'GPT-4-Turbo-2024-04-09': 'gpt-4-turbo-2024-04-09', +// 'GPT-4-1106-preview': 'gpt-4-1106-preview', +// 'GPT-4-0125-preview': 'gpt-4-0125-preview', +// 'GPT-4-0314': 'gpt-4-0314', +// 'GPT-4-0613': 'gpt-4-0613', +// 'GPT-3.5-Turbo-0613': 'gpt-3.5-turbo-0613', +// 'GPT-3.5-Turbo-0314': 'gpt-3.5-turbo-0314', +// 'GPT-3.5-Turbo-0125': 'gpt-3.5-turbo-0125', +// +// // Mistral models (from the previous file) +// 'Mistral-Large-2402': 'mistral-large-2402', +// 'Mixtral-8x7b-Instruct-v0.1': 'mistralai/Mixtral-8x7B-Instruct-v0.1', +// +// // Other models without matches +// 'Gemini-1.5-Pro-Exp-0801': '', +// 'Meta-Llama-3.1-405b-Instruct': '', +// 'Gemini-1.5-Pro-001': '', +// 'Meta-Llama-3.1-70b-Instruct': '', +// 'Yi-Large-preview': '', +// 'Deepseek-v2-API-0628': '', +// 'Gemma-2-27b-it': '', +// 'Yi-Large': '', +// 'Nemotron-4-340B-Instruct': '', +// 'GLM-4-0520': '', +// 'Llama-3-70b-Instruct': '', +// 'Reka-Core-20240501': '', +// 'Command R+': '', +// 'Gemma-2-9b-it': '', +// 'Qwen2-72B-Instruct': '', +// 'GLM-4-0116': '', +// 'Qwen-Max-0428': '', +// 'DeepSeek-Coder-V2-Instruct': '', +// 'Reka-Flash-Preview-20240611': '', +// 'Meta-Llama-3.1-8b-Instruct': '', +// 'Qwen1.5-110B-Chat': '', +// 'Yi-1.5-34B-Chat': '', +// 'Reka-Flash-21B-online': '', +// 'Llama-3-8b-Instruct': '', +// 'Command R': '', +// 'Reka-Flash-21B': '', +// 'Qwen1.5-72B-Chat': '', +// 'Mixtral-8x22b-Instruct-v0.1': '', +// 'Zephyr-ORPO-141b-A35b-v0.1': '', +// 'Qwen1.5-32B-Chat': '', +// 'Mistral-Next': '', +// 'Phi-3-Medium-4k-Instruct': '', +// 'Starling-LM-7B-beta': '', +// 'Yi-34B-Chat': '', +// 'Qwen1.5-14B-Chat': '', +// 'WizardLM-70B-v1.0': '', +// 'Tulu-2-DPO-70B': '', +// 'DBRX-Instruct-Preview': '', +// 'Phi-3-Small-8k-Instruct': '', +// 'Llama-2-70b-chat': '', +// 'OpenChat-3.5-0106': '', +// 'Vicuna-33B': '', +// 'Snowflake Arctic Instruct': '', +// 'Starling-LM-7B-alpha': '', +// }; \ No newline at end of file From dec280d54d1819c2fa95efd78a8903ffbba25621 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Wed, 7 Aug 2024 02:50:10 -0700 Subject: [PATCH 3/3] 1.16.7 Release (cherry picked from commit 22b32d571d8455818362492cafbd8ca1070c46f9) --- README.md | 3 ++- docs/changelog.md | 3 ++- src/apps/news/news.data.tsx | 3 ++- src/apps/news/news.version.tsx | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6a1202b39..387049d3a 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,9 @@ Or fork & run on Vercel [//]: # (big-AGI is an open book; see the **[ready-to-ship and future ideas](https://github.com/users/enricoros/projects/4/views/2)** in our open roadmap) -### What's New in 1.16.1...1.16.6 · Jul 25, 2024 (patch releases) +### What's New in 1.16.1...1.16.7 · Aug 7, 2024 (patch releases) +- 1.16.7: OpenAI support for GPT-4o 2024-08-06 - 1.16.6: Groq support for Llama 3.1 models - 1.16.5: GPT-4o Mini support - 1.16.4: 8192 tokens support for Claude 3.5 Sonnet diff --git a/docs/changelog.md b/docs/changelog.md index 2ce8e7fff..40723688b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,8 +10,9 @@ by release. - milestone: [1.17.0](https://github.com/enricoros/big-agi/milestone/17) - work in progress: [big-AGI open roadmap](https://github.com/users/enricoros/projects/4/views/2), [help here](https://github.com/users/enricoros/projects/4/views/4) -### What's New in 1.16.1...1.16.6 · Jul 25, 2024 (patch releases) +### What's New in 1.16.1...1.16.7 · Aug 7, 2024 (patch releases) +- 1.16.7: OpenAI support for GPT-4o 2024-08-06 - 1.16.6: Groq support for Llama 3.1 models - 1.16.5: GPT-4o Mini support - 1.16.4: 8192 tokens support for Claude 3.5 Sonnet diff --git a/src/apps/news/news.data.tsx b/src/apps/news/news.data.tsx index 8dff6dc7d..ff3e0b957 100644 --- a/src/apps/news/news.data.tsx +++ b/src/apps/news/news.data.tsx @@ -61,7 +61,7 @@ export const NewsItems: NewsItem[] = [ ] }*/ { - versionCode: '1.16.6', + versionCode: '1.16.7', versionName: 'Crystal Clear', versionDate: new Date('2024-06-07T05:00:00Z'), // versionDate: new Date('2024-05-13T19:00:00Z'), @@ -83,6 +83,7 @@ export const NewsItems: NewsItem[] = [ { text: <>1.16.4: 8192 tokens support for Claude 3.5 Sonnet }, { text: <>1.16.5: OpenAI GPT-4o Mini support }, { text: <>1.16.6: Groq Llama 3.1 support }, + { text: <>1.16.7: Gpt-4o 2024-08-06 }, ], }, { diff --git a/src/apps/news/news.version.tsx b/src/apps/news/news.version.tsx index 12cece6b3..d2d3ad3aa 100644 --- a/src/apps/news/news.version.tsx +++ b/src/apps/news/news.version.tsx @@ -7,7 +7,7 @@ import { useAppStateStore } from '~/common/state/store-appstate'; // update this variable every time you want to broadcast a new version to clients -export const incrementalNewsVersion: number = 16.1; // not notifying for 1.16.6 +export const incrementalNewsVersion: number = 16.1; // not notifying for 1.16.7 interface NewsState {