[feat] oaifixer

src/lib/Setting/Pages/PluginSettings.svelte

@@ -25,17 +25,25 @@
         <Check bind:check={$DataBase.officialplugins.automark} name={language.able}/>
     </div>
     <div class="flex">
-        <span class="font-bold flex-grow">Romanizer <Help key="romanizer" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
+        <span class="font-bold flex-grow">Metric Systemizer <Help key="metrica" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
     </div>
     <div class="flex items-center mt-2">
-        <Check bind:check={$DataBase.officialplugins.romanizer} name={language.able}/>
+        <Check bind:check={$DataBase.officialplugins.metrica} name={language.able}/>
     </div>
     <div class="flex">
-        <span class="font-bold flex-grow">Metric Systemizer <Help key="metrica" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
+        <span class="font-bold flex-grow">OpenAI Fixer <Help key="metrica" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
     </div>
     <div class="flex items-center mt-2">
-        <Check bind:check={$DataBase.officialplugins.metrica} name={language.able}/>
+        <Check bind:check={$DataBase.officialplugins.oaiFix} name={language.able}/>
     </div>
+    {#if $DataBase.officialplugins.oaiFix}
+        <div class="flex items-center mt-2">
+            <Check bind:check={$DataBase.officialplugins.oaiFix} name={"Remove Emdash"}/>
+        </div>
+        <div class="flex items-center mt-2">
+            <Check bind:check={$DataBase.officialplugins.oaiFixLetters} name={"Fix Letters"}/>
+        </div>
+    {/if}
     {#each $DataBase.plugins as plugin, i}
         <div class="border-borderc mt-2 mb-2 w-full border-solid border-b-1 seperator"></div>
         <div class="flex">

src/ts/plugins/fixer.ts

@@ -0,0 +1,43 @@
+export function OaifixEmdash(bias:{[key:number]:number}){
+    const emdashes = [
+        2001,  2345,  8713, 16620, 17223,
+       22416, 29096, 29472, 30697, 35192,
+       38542, 41128, 44603, 49525, 50004,
+       50617, 51749, 51757, 55434, 60654,
+       61311, 63750, 63938, 63977, 66101,
+       68850, 71201, 71480, 72318, 76070,
+       76929, 80078, 81902, 83872, 84941,
+       85366, 86319, 87247, 87671, 88958,
+       90863, 93830, 96197, 99563
+    ]
+
+    for (const emdash of emdashes) {
+        bias[emdash] = -100
+    }
+
+    return bias
+
+}
+
+export function OaiFixKorean(text:string){
+
+    //tokenizer problem fixes
+    const replacer = {
+        //commonly wrong english
+        '피츠': '피스',
+        '스커츠': '스커트',
+        '스파츠': '스커트',
+        '스마트폰': '스파트폰',
+        '스위츠': '스위치',
+        '해도 되': '해도 돼',
+        '해도 됩니다': '해도 돼요',
+        '에레베이터': '엘리베이터',
+        '에리베이터': '엘리베이터',
+        '에레바토르': '엘리베이터',
+    }
+
+    for (const key in replacer) {
+        text = text.replace(key, replacer[key])
+    }
+    return text
+}

src/ts/process/index.ts

@@ -557,29 +557,6 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
         currentTokens += await tokenizer.tokenizeChat(chat)
     }
 
-    if(db.officialplugins.romanizer){
-        const romanizer = await import('../plugins/romanizer')
-        const r = romanizer.romanizer(chats.map((v) => {
-            return v.content   
-        }))
-
-        for(let i=0;i<chats.length;i++){
-            const pchat = cloneDeep(chats[i])
-            pchat.content = r.result[i]
-            if(await tokenizer.tokenizeChat(chats[i]) > await tokenizer.tokenizeChat(pchat)){
-                chats[i] = pchat
-            }
-        }
-
-        if(r.mostUsed !== 'roman'){
-
-            unformated.postEverything.push({
-                role: 'system',
-                content: `user and assistant are chatting with romanized ${r.mostUsed}, but always respond with ${r.mostUsed} with ${r.mostUsed} letters.`
-            })
-        }
-    }
-
     if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
         const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
             asHyper: db.supaMemoryType !== 'subModel' && db.hypaMemory

src/ts/process/request.ts

@@ -18,6 +18,7 @@ import { Sha256 } from "@aws-crypto/sha256-js";
 import { v4 } from "uuid";
 import { cloneDeep } from "lodash";
 import { supportsInlayImage } from "../image";
+import { OaifixEmdash } from "../plugins/fixer";
 
 
 
@@ -236,6 +237,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 }
             }
 
+            if(db.officialplugins.oaiFix && db.officialplugins.oaiFixEmdash){
+                if(raiModel.startsWith('gpt35') || raiModel.startsWith('gpt4')){
+                    bias = OaifixEmdash(bias)
+                }
+            }
+
 
 
 
@@ -273,7 +280,6 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
             }
 
 
-            console.log(bias)
             db.cipherChat = false
             let body = ({
                 model: aiModel === 'openrouter' ? db.openrouterRequestModel :

src/ts/process/scripts.ts

@@ -9,6 +9,7 @@ import { risuChatParser as risuChatParserOrg, type simpleCharacterArgument } fro
 import { autoMarkPlugin } from "../plugins/automark";
 import { runCharacterJS } from "../plugins/embedscript";
 import { metricaPlugin } from "../plugins/metrica";
+import { OaiFixKorean } from "../plugins/fixer";
 
 const dreg = /{{data}}/g
 const randomness = /\|\|\|/g
@@ -69,6 +70,9 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter
     if(db.officialplugins.metrica && (mode === 'editinput' || mode === 'editoutput' || mode === 'editprocess')){
         data = metricaPlugin(data, 'imperial')
     }
+    if(db.officialplugins.oaiFixLetters && db.officialplugins.oaiFix && (mode === 'editoutput' || mode === 'editdisplay')){
+        data = OaiFixKorean(data)
+    }
     data = await runCharacterJS({
         code: char.virtualscript ?? null,
         mode,

src/ts/storage/database.ts

@@ -396,6 +396,9 @@ export interface Database{
         automark?: boolean
         romanizer?: boolean
         metrica?: boolean
+        oaiFix?: boolean
+        oaiFixEmdash?: boolean
+        oaiFixLetters?: boolean
     }
     currentPluginProvider: string
     zoomsize:number