Merge pull request #28 from ubq-testing/fix/missing-context

Fix/missing context

.cspell.json

@@ -34,7 +34,11 @@
     "Typeguard",
     "typeguards",
     "OPENROUTER_API_KEY",
-    "Openrouter"
+    "Openrouter",
+    "flac",
+    "dylib",
+    "mobileprovision",
+    "icns"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

package.json

@@ -34,7 +34,6 @@
     "@supabase/supabase-js": "^2.45.4",
     "@ubiquity-os/ubiquity-os-logger": "^1.3.2",
     "dotenv": "^16.4.5",
-    "github-diff-tool": "^1.0.6",
     "gpt-tokenizer": "^2.5.1",
     "openai": "^4.63.0",
     "typebox-validators": "0.3.5",

src/adapters/openai/helpers/completions.ts

@@ -3,6 +3,7 @@ import { Context } from "../../../types";
 import { SuperOpenAi } from "./openai";
 import { CompletionsModelHelper, ModelApplications } from "../../../types/llm";
 import { encode } from "gpt-tokenizer";
+import { logger } from "../../../helpers/errors";
 
 export interface CompletionsType {
   answer: string;
@@ -22,15 +23,69 @@ export class Completions extends SuperOpenAi {
     this.context = context;
   }
 
+  getModelMaxTokenLimit(model: string): number {
+    // could be made more robust, unfortunately, there's no endpoint to get the model token limit
+    const tokenLimits = new Map<string, number>([
+      ["o1-mini", 128_000],
+      ["o1-preview", 128_000],
+      ["gpt-4-turbo", 128_000],
+      ["gpt-4o", 128_000],
+      ["gpt-4o-mini", 128_000],
+      ["gpt-4", 8_192],
+      ["gpt-3.5-turbo-0125", 16_385],
+      ["gpt-3.5-turbo", 16_385],
+    ]);
+
+    return tokenLimits.get(model) || 128_000;
+  }
+
+  getModelMaxOutputLimit(model: string): number {
+    // could be made more robust, unfortunately, there's no endpoint to get the model token limit
+    const tokenLimits = new Map<string, number>([
+      ["o1-mini", 65_536],
+      ["o1-preview", 32_768],
+      ["gpt-4-turbo", 4_096],
+      ["gpt-4o-mini", 16_384],
+      ["gpt-4o", 16_384],
+      ["gpt-4", 8_192],
+      ["gpt-3.5-turbo-0125", 4_096],
+      ["gpt-3.5-turbo", 4_096],
+    ]);
+
+    return tokenLimits.get(model) || 16_384;
+  }
+
+  async getModelTokenLimit(): Promise<number> {
+    return this.getModelMaxTokenLimit("o1-mini");
+  }
+
   async createCompletion(
-    prompt: string,
+    query: string,
     model: string = "o1-mini",
     additionalContext: string[],
     localContext: string[],
     groundTruths: string[],
     botName: string,
     maxTokens: number
   ): Promise<CompletionsType> {
+    const numTokens = await this.findTokenLength(query, additionalContext, localContext, groundTruths);
+    logger.info(`Number of tokens: ${numTokens}`);
+
+    const sysMsg = [
+      "You Must obey the following ground truths: ",
+      JSON.stringify(groundTruths) + "\n",
+      "You are tasked with assisting as a GitHub bot by generating responses based on provided chat history and similar responses, focusing on using available knowledge within the provided corpus, which may contain code, documentation, or incomplete information. Your role is to interpret and use this knowledge effectively to answer user questions.\n\n# Steps\n\n1. **Understand Context**: Review the chat history and any similar provided responses to understand the context.\n2. **Extract Relevant Information**: Identify key pieces of information, even if they are incomplete, from the available corpus.\n3. **Apply Knowledge**: Use the extracted information and relevant documentation to construct an informed response.\n4. **Draft Response**: Compile the gathered insights into a coherent and concise response, ensuring it's clear and directly addresses the user's query.\n5. **Review and Refine**: Check for accuracy and completeness, filling any gaps with logical assumptions where necessary.\n\n# Output Format\n\n- Concise and coherent responses in paragraphs that directly address the user's question.\n- Incorporate inline code snippets or references from the documentation if relevant.\n\n# Examples\n\n**Example 1**\n\n*Input:*\n- Chat History: \"What was the original reason for moving the LP tokens?\"\n- Corpus Excerpts: \"It isn't clear to me if we redid the staking yet and if we should migrate. If so, perhaps we should make a new issue instead. We should investigate whether the missing LP tokens issue from the MasterChefV2.1 contract is critical to the decision of migrating or not.\"\n\n*Output:*\n\"It was due to missing LP tokens issue from the MasterChefV2.1 Contract.\n\n# Notes\n\n- Ensure the response is crafted from the corpus provided, without introducing information outside of what's available or relevant to the query.\n- Consider edge cases where the corpus might lack explicit answers, and justify responses with logical reasoning based on the existing information.",
+      `Your name is: ${botName}`,
+      "\n",
+      "Main Context (Provide additional precedence in terms of information): ",
+      localContext.join("\n"),
+      "Secondary Context: ",
+      additionalContext.join("\n"),
+    ].join("\n");
+
+    logger.info(`System message: ${sysMsg}`);
+    logger.info(`Query: ${query}`);
+
     const res: OpenAI.Chat.Completions.ChatCompletion = await this.client.chat.completions.create({
       model: model,
       messages: [
@@ -39,18 +94,7 @@ export class Completions extends SuperOpenAi {
           content: [
             {
               type: "text",
-              text:
-                "You Must obey the following ground truths: [" +
-                groundTruths.join(":") +
-                "]\n" +
-                "You are tasked with assisting as a GitHub bot by generating responses based on provided chat history and similar responses, focusing on using available knowledge within the provided corpus, which may contain code, documentation, or incomplete information. Your role is to interpret and use this knowledge effectively to answer user questions.\n\n# Steps\n\n1. **Understand Context**: Review the chat history and any similar provided responses to understand the context.\n2. **Extract Relevant Information**: Identify key pieces of information, even if they are incomplete, from the available corpus.\n3. **Apply Knowledge**: Use the extracted information and relevant documentation to construct an informed response.\n4. **Draft Response**: Compile the gathered insights into a coherent and concise response, ensuring it's clear and directly addresses the user's query.\n5. **Review and Refine**: Check for accuracy and completeness, filling any gaps with logical assumptions where necessary.\n\n# Output Format\n\n- Concise and coherent responses in paragraphs that directly address the user's question.\n- Incorporate inline code snippets or references from the documentation if relevant.\n\n# Examples\n\n**Example 1**\n\n*Input:*\n- Chat History: \"What was the original reason for moving the LP tokens?\"\n- Corpus Excerpts: \"It isn't clear to me if we redid the staking yet and if we should migrate. If so, perhaps we should make a new issue instead. We should investigate whether the missing LP tokens issue from the MasterChefV2.1 contract is critical to the decision of migrating or not.\"\n\n*Output:*\n\"It was due to missing LP tokens issue from the MasterChefV2.1 Contract.\n\n# Notes\n\n- Ensure the response is crafted from the corpus provided, without introducing information outside of what's available or relevant to the query.\n- Consider edge cases where the corpus might lack explicit answers, and justify responses with logical reasoning based on the existing information." +
-                "Your name is : " +
-                botName +
-                "\n" +
-                "Main Context (Provide additional precedence in terms of information): " +
-                localContext.join("\n") +
-                "Secondary Context: " +
-                additionalContext.join("\n"),
+              text: sysMsg,
             },
           ],
         },
@@ -59,7 +103,7 @@ export class Completions extends SuperOpenAi {
           content: [
             {
               type: "text",
-              text: prompt,
+              text: query,
             },
           ],
         },
@@ -73,6 +117,7 @@ export class Completions extends SuperOpenAi {
         type: "text",
       },
     });
+
     const answer = res.choices[0].message;
     if (answer && answer.content && res.usage) {
       return {
@@ -120,6 +165,7 @@ export class Completions extends SuperOpenAi {
   }
 
   async findTokenLength(prompt: string, additionalContext: string[] = [], localContext: string[] = [], groundTruths: string[] = []): Promise<number> {
-    return encode(prompt + additionalContext.join("\n") + localContext.join("\n") + groundTruths.join("\n")).length;
+    // disallowedSpecial: new Set() because we pass the entire diff as the prompt we should account for all special characters
+    return encode(prompt + additionalContext.join("\n") + localContext.join("\n") + groundTruths.join("\n"), { disallowedSpecial: new Set() }).length;
   }
 }

src/handlers/ask-llm.ts

@@ -6,37 +6,26 @@ import { recursivelyFetchLinkedIssues } from "../helpers/issue-fetching";
 import { formatChatHistory } from "../helpers/format-chat-history";
 import { fetchRepoDependencies, fetchRepoLanguageStats } from "./ground-truths/chat-bot";
 import { findGroundTruths } from "./ground-truths/find-ground-truths";
-import { bubbleUpErrorComment } from "../helpers/errors";
+import { bubbleUpErrorComment, logger } from "../helpers/errors";
 
-/**
- * Asks a question to GPT and returns the response
- * @param context - The context object containing environment and configuration details
- * @param question - The question to ask GPT
- * @returns The response from GPT
- * @throws If no question is provided
- */
 export async function askQuestion(context: Context, question: string) {
   if (!question) {
-    throw context.logger.error("No question provided");
+    throw logger.error("No question provided");
   }
+  // using any links in comments or issue/pr bodies to fetch more context
   const { specAndBodies, streamlinedComments } = await recursivelyFetchLinkedIssues({
     context,
     owner: context.payload.repository.owner.login,
     repo: context.payload.repository.name,
   });
+  // build a nicely structure system message containing a streamlined chat history
+  // includes the current issue, any linked issues, and any linked PRs
   const formattedChat = await formatChatHistory(context, streamlinedComments, specAndBodies);
-  context.logger.info(`${formattedChat.join("")}`);
-  return await askGpt(context, question, formattedChat);
+  logger.info(`${formattedChat.join("")}`);
+  return await askLlm(context, question, formattedChat);
 }
 
-/**
- * Asks GPT a question and returns the completions
- * @param context - The context object containing environment and configuration details
- * @param question - The question to ask GPT
- * @param formattedChat - The formatted chat history to provide context to GPT
- * @returns completions - The completions generated by GPT
- **/
-export async function askGpt(context: Context, question: string, formattedChat: string[]): Promise<CompletionsType> {
+export async function askLlm(context: Context, question: string, formattedChat: string[]): Promise<CompletionsType> {
   const {
     env: { UBIQUITY_OS_APP_NAME },
     config: { model, similarityThreshold, maxTokens },
@@ -45,31 +34,49 @@ export async function askGpt(context: Context, question: string, formattedChat:
       voyage: { reranker },
       openai: { completions },
     },
-    logger,
   } = context;
 
   try {
+    // using db functions to find similar comments and issues
     const [similarComments, similarIssues] = await Promise.all([
       comment.findSimilarComments(question, 1 - similarityThreshold, ""),
       issue.findSimilarIssues(question, 1 - similarityThreshold, ""),
     ]);
 
+    // combine the similar comments and issues into a single array
     const similarText = [
       ...(similarComments?.map((comment: CommentSimilaritySearchResult) => comment.comment_plaintext) || []),
       ...(similarIssues?.map((issue: IssueSimilaritySearchResult) => issue.issue_plaintext) || []),
     ];
 
+    // filter out any empty strings
     formattedChat = formattedChat.filter((text) => text);
 
+    // rerank the similar text using voyageai
     const rerankedText = similarText.length > 0 ? await reranker.reRankResults(similarText, question) : [];
+    // gather structural data about the payload repository
     const [languages, { dependencies, devDependencies }] = await Promise.all([fetchRepoLanguageStats(context), fetchRepoDependencies(context)]);
 
-    const groundTruths = await findGroundTruths(context, "chat-bot", { languages, dependencies, devDependencies });
+    let groundTruths: string[] = [];
 
-    const numTokens = await completions.findTokenLength(question, rerankedText, formattedChat, groundTruths);
-    logger.info(`Number of tokens: ${numTokens}`);
+    if (!languages.length) {
+      groundTruths.push("No languages found in the repository");
+    }
 
-    return completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME, maxTokens);
+    if (!Reflect.ownKeys(dependencies).length) {
+      groundTruths.push("No dependencies found in the repository");
+    }
+
+    if (!Reflect.ownKeys(devDependencies).length) {
+      groundTruths.push("No devDependencies found in the repository");
+    }
+
+    if (groundTruths.length === 3) {
+      return await completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME, maxTokens);
+    }
+
+    groundTruths = await findGroundTruths(context, "chat-bot", { languages, dependencies, devDependencies });
+    return await completions.createCompletion(question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME, maxTokens);
   } catch (error) {
     throw bubbleUpErrorComment(context, error, false);
   }

src/handlers/comment-created-callback.ts

@@ -27,24 +27,25 @@ export async function issueCommentCreatedCallback(
     return { status: 204, reason: logger.info("Comment is from a bot. Skipping.").logMessage.raw };
   }
 
-  logger.info(`Asking question: ${question}`);
-  let commentToPost;
   try {
     const response = await askQuestion(context, question);
     const { answer, tokenUsage, groundTruths } = response;
     if (!answer) {
       throw logger.error(`No answer from OpenAI`);
     }
-    logger.info(`Answer: ${answer}`, { tokenUsage });
 
-    const metadata = {
-      groundTruths,
-      tokenUsage,
-    };
+    const metadataString = createStructuredMetadata(
+      // don't change this header, it's used for tracking
+      "ubiquity-os-llm-response",
+      logger.info(`Answer: ${answer}`, {
+        metadata: {
+          groundTruths,
+          tokenUsage,
+        },
+      })
+    );
 
-    const metadataString = createStructuredMetadata("LLM Ground Truths and Token Usage", logger.info(`Answer: ${answer}`, { metadata }));
-    commentToPost = answer + metadataString;
-    await addCommentToIssue(context, commentToPost);
+    await addCommentToIssue(context, answer + metadataString);
     return { status: 200, reason: logger.info("Comment posted successfully").logMessage.raw };
   } catch (error) {
     throw await bubbleUpErrorComment(context, error, false);

src/handlers/comments.ts

@@ -10,11 +10,14 @@ import { StreamlinedComment } from "../types/llm";
  */
 export async function getAllStreamlinedComments(linkedIssues: LinkedIssues[]) {
   const streamlinedComments: Record<string, StreamlinedComment[]> = {};
+
   for (const issue of linkedIssues) {
     const linkedIssueComments = issue.comments || [];
     if (linkedIssueComments.length === 0) continue;
+
     const linkedStreamlinedComments = streamlineComments(linkedIssueComments);
     if (!linkedStreamlinedComments) continue;
+
     for (const [key, value] of Object.entries(linkedStreamlinedComments)) {
       streamlinedComments[key] = [...(streamlinedComments[key] || []), ...value];
     }
@@ -74,15 +77,15 @@ export function createKey(issueUrl: string, issue?: number) {
  */
 export function streamlineComments(comments: SimplifiedComment[]) {
   const streamlined: Record<string, StreamlinedComment[]> = {};
+
   for (const comment of comments) {
     const { user, issueUrl: url, body } = comment;
-    // Skip bot comments
     if (user?.type === "Bot") continue;
+
     const key = createKey(url);
     const [owner, repo] = splitKey(key);
-    if (!streamlined[key]) {
-      streamlined[key] = [];
-    }
+    streamlined[key] ??= [];
+
     if (user && body) {
       streamlined[key].push({
         user: user.login,

src/handlers/ground-truths/chat-bot.ts

@@ -68,6 +68,7 @@ export async function fetchRepoLanguageStats(context: Context) {
 
     return Array.from(Object.entries(stats)).sort((a, b) => b[1] - a[1]);
   } catch (err) {
-    throw logger.error(`Error fetching language stats for ${owner}/${repo}`, { err });
+    logger.error(`Error fetching language stats for ${owner}/${repo}`, { err });
+    return [];
   }
 }