From 6e4b5c14fe537b1655650523dbe92b241558fdf9 Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 13:07:33 +0100 Subject: [PATCH 1/7] Use the description if the author hasn't added an assertion --- src/github/index.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/github/index.ts b/src/github/index.ts index d33cc4c..3ebb3c8 100644 --- a/src/github/index.ts +++ b/src/github/index.ts @@ -69,14 +69,36 @@ export const getSinglePR = async ( } }; + +// If we can get a PR, we can parse the description and try to isolate the description +export const getDescription = (description: string): string | null => { + // find everything after ## Description and before the next ## heading + const regexDescription = /(?<=## Description)(.*?)(?=##|$)/gm + const regexBetweenComments = /(?<=-->)(.*?)(?= and const regex = /([\s\S]*?)/g; const assertion = regex.exec(description); + const descriptionActual = getDescription(description); if (assertion) { console.log(`✅ Got assertion: ${assertion[1]}`); return assertion[1]; + } else if (descriptionActual) { + console.log(`✅ Got assertion from description: ${descriptionActual}`); + return descriptionActual; } return null; }; From d96e31fd9ff5fbdb2ce3f2df01aca0f13569733a Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 13:23:10 +0100 Subject: [PATCH 2/7] Functions do one thing only amen --- src/github/index.ts | 4 ---- src/index.ts | 33 ++++++++++++++++++--------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/github/index.ts b/src/github/index.ts index 3ebb3c8..695fc33 100644 --- a/src/github/index.ts +++ b/src/github/index.ts @@ -92,13 +92,9 @@ export const getAssertion = async (description: string): Promise // find everything in between and const regex = /([\s\S]*?)/g; const assertion = regex.exec(description); - const descriptionActual = getDescription(description); if (assertion) { console.log(`✅ Got assertion: ${assertion[1]}`); return assertion[1]; - } else if (descriptionActual) { - console.log(`✅ Got assertion from description: ${descriptionActual}`); - return descriptionActual; } return null; }; diff --git a/src/index.ts b/src/index.ts index 77640ff..5797761 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import dotenv from 'dotenv'; import * as core from '@actions/core'; -import { getSinglePR, getAssertion, getDiff, getChangedFiles, getFileContent } from './github'; +import { getSinglePR, getAssertion, getDiff, getChangedFiles, getFileContent, getDescription } from './github'; import { generatePrompt, testAssertion, writeAnalysis, openAiFeedback } from './open_ai'; dotenv.config(); @@ -16,20 +16,23 @@ const repoName = repo.split('/')[1]; async function main() { const PR = await getSinglePR(org, repoName, prNumber); const assertion = await getAssertion(PR?.body ?? ''); - if (assertion?.length === 0 || assertion === null) { - console.log('No assertion found'); - core.setFailed('No assertion found'); - return; - } else { - const diff: string = await getDiff(prNumber, org, repoName); - const changedFiles = getChangedFiles(diff); - const file: any = await getFileContent(changedFiles, org, repoName); - const prompt: string = generatePrompt(diff, assertion, file); - const rawAnalysis = await testAssertion(prompt); - const analysis = writeAnalysis(rawAnalysis); - console.log(analysis); - core.setOutput('analysis', analysis); - return analysis; + if (!assertion) { + const description = getDescription(PR?.body ?? ''); + if (!description) { + console.log('No assertion or description found'); + core.setFailed('No assertion or description found'); + return; + } else { + const diff: string = await getDiff(prNumber, org, repoName); + const changedFiles = getChangedFiles(diff); + const file: any = await getFileContent(changedFiles, org, repoName); + const prompt: string = generatePrompt(diff, assertion, file); + const rawAnalysis = await testAssertion(prompt); + const analysis = writeAnalysis(rawAnalysis); + console.log(analysis); + core.setOutput('analysis', analysis); + return analysis; + } } } From a58d52e45c437d2cc75e9de133e5f08f877603da Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 16:07:42 +0100 Subject: [PATCH 3/7] Fix logic --- src/index.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/index.ts b/src/index.ts index 5797761..f9b6748 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,25 +15,25 @@ const repoName = repo.split('/')[1]; async function main() { const PR = await getSinglePR(org, repoName, prNumber); - const assertion = await getAssertion(PR?.body ?? ''); - if (!assertion) { - const description = getDescription(PR?.body ?? ''); - if (!description) { + // Try use actual assertion, if not use description + let assertionToUse = await getAssertion(PR?.body ?? ''); + if (!assertionToUse) { + assertionToUse = getDescription(PR?.body ?? ''); + if (!assertionToUse) { console.log('No assertion or description found'); core.setFailed('No assertion or description found'); return; - } else { - const diff: string = await getDiff(prNumber, org, repoName); - const changedFiles = getChangedFiles(diff); - const file: any = await getFileContent(changedFiles, org, repoName); - const prompt: string = generatePrompt(diff, assertion, file); - const rawAnalysis = await testAssertion(prompt); - const analysis = writeAnalysis(rawAnalysis); - console.log(analysis); - core.setOutput('analysis', analysis); - return analysis; } } + const diff: string = await getDiff(prNumber, org, repoName); + const changedFiles = getChangedFiles(diff); + const file: any = await getFileContent(changedFiles, org, repoName); + const prompt: string = generatePrompt(diff, assertionToUse, file); + const rawAnalysis = await testAssertion(prompt); + const analysis = writeAnalysis(rawAnalysis); + console.log(analysis); + core.setOutput('analysis', analysis); + return analysis; } main(); From c98697f14437a8918430193923f8ad3f9723d937 Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 16:16:29 +0100 Subject: [PATCH 4/7] =?UTF-8?q?Apply=20some=20optimizations=20which=20chat?= =?UTF-8?q?GPT=20suggested=20=F0=9F=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/index.ts | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/index.ts b/src/index.ts index f9b6748..cb6efbc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,34 +6,39 @@ import { generatePrompt, testAssertion, writeAnalysis, openAiFeedback } from './ dotenv.config(); // Our configuration variables using GitHub Actions for production and dotenv for local development -const prNumber: number = parseInt(core.getInput('PR_NUMBER') || (process.env.PR_NUMBER as string)); -const org: string = core.getInput('GITHUB_ORG') || (process.env.GITHUB_ORG as string); -const repo: string = core.getInput('GITHUB_REPOSITORY') || (process.env.GITHUB_REPOSITORY as string); +const prNumber: number = parseInt(core.getInput('PR_NUMBER') || process.env.PR_NUMBER || ''); +const org: string = core.getInput('GITHUB_ORG') || process.env.GITHUB_ORG || ''; +const repo: string = core.getInput('GITHUB_REPOSITORY') || process.env.GITHUB_REPOSITORY || ''; // We'll need to parse the repo variable to remove the owner and the / from the string const repoName = repo.split('/')[1]; async function main() { - const PR = await getSinglePR(org, repoName, prNumber); - // Try use actual assertion, if not use description - let assertionToUse = await getAssertion(PR?.body ?? ''); - if (!assertionToUse) { - assertionToUse = getDescription(PR?.body ?? ''); + try { + const PR = await getSinglePR(org, repoName, prNumber); + + // Try to use actual assertion, if not use description + let assertionToUse: string | null = await getAssertion(PR?.body ?? ''); if (!assertionToUse) { - console.log('No assertion or description found'); - core.setFailed('No assertion or description found'); - return; + assertionToUse = getDescription(PR?.body ?? ''); + if (!assertionToUse) { + core.setFailed('No assertion or description found'); + return; + } } + + const diff: string = await getDiff(prNumber, org, repoName); + const changedFiles: string[] = getChangedFiles(diff); + const file: any = await getFileContent(changedFiles, org, repoName); + const prompt: string = generatePrompt(diff, assertionToUse, file); + const rawAnalysis = await testAssertion(prompt); + const analysis: string = writeAnalysis(rawAnalysis); + console.log(analysis); + core.setOutput('analysis', analysis); + core.info('Analysis completed successfully'); + } catch (error) { + core.setFailed(`Error during execution: ${(error as Error).message}`); } - const diff: string = await getDiff(prNumber, org, repoName); - const changedFiles = getChangedFiles(diff); - const file: any = await getFileContent(changedFiles, org, repoName); - const prompt: string = generatePrompt(diff, assertionToUse, file); - const rawAnalysis = await testAssertion(prompt); - const analysis = writeAnalysis(rawAnalysis); - console.log(analysis); - core.setOutput('analysis', analysis); - return analysis; } main(); From a9083ea72b0114024d4d0afc9951691495b179f8 Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 16:24:40 +0100 Subject: [PATCH 5/7] Updates model to gpt-4o. Cheaper, faster, stronger, harder. --- src/open_ai/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/open_ai/index.ts b/src/open_ai/index.ts index df7c52d..f7f86d2 100644 --- a/src/open_ai/index.ts +++ b/src/open_ai/index.ts @@ -85,7 +85,7 @@ export const testAssertion = async (prompt: string): Promise Date: Thu, 13 Jun 2024 16:28:46 +0100 Subject: [PATCH 6/7] Build --- dist/github/index.js | 18 ++++++++++++++- dist/index.js | 52 ++++++++++++++++++++++++++++++------------- dist/open_ai/index.js | 2 +- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/dist/github/index.js b/dist/github/index.js index 47e7780..b90cf86 100644 --- a/dist/github/index.js +++ b/dist/github/index.js @@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0; +exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getDescription = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0; const dotenv_1 = __importDefault(require("dotenv")); const core = __importStar(require("@actions/core")); const rest_1 = require("@octokit/rest"); @@ -87,6 +87,22 @@ const getSinglePR = async (owner, repo, prNumber) => { } }; exports.getSinglePR = getSinglePR; +// If we can get a PR, we can parse the description and try to isolate the description +const getDescription = (description) => { + // find everything after ## Description and before the next ## heading + const regexDescription = /(?<=## Description)(.*?)(?=##|$)/gm; + const regexBetweenComments = /(?<=-->)(.*?)(?= and diff --git a/dist/index.js b/dist/index.js index f09bcda..5eda2a9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -33,7 +33,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0; +exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getDescription = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0; const dotenv_1 = __importDefault(__nccwpck_require__(2437)); const core = __importStar(__nccwpck_require__(2186)); const rest_1 = __nccwpck_require__(5375); @@ -94,6 +94,22 @@ const getSinglePR = async (owner, repo, prNumber) => { } }; exports.getSinglePR = getSinglePR; +// If we can get a PR, we can parse the description and try to isolate the description +const getDescription = (description) => { + // find everything after ## Description and before the next ## heading + const regexDescription = /(?<=## Description)(.*?)(?=##|$)/gm; + const regexBetweenComments = /(?<=-->)(.*?)(?= and @@ -200,29 +216,35 @@ const github_1 = __nccwpck_require__(4321); const open_ai_1 = __nccwpck_require__(2580); dotenv_1.default.config(); // Our configuration variables using GitHub Actions for production and dotenv for local development -const prNumber = parseInt(core.getInput('PR_NUMBER') || process.env.PR_NUMBER); -const org = core.getInput('GITHUB_ORG') || process.env.GITHUB_ORG; -const repo = core.getInput('GITHUB_REPOSITORY') || process.env.GITHUB_REPOSITORY; +const prNumber = parseInt(core.getInput('PR_NUMBER') || process.env.PR_NUMBER || ''); +const org = core.getInput('GITHUB_ORG') || process.env.GITHUB_ORG || ''; +const repo = core.getInput('GITHUB_REPOSITORY') || process.env.GITHUB_REPOSITORY || ''; // We'll need to parse the repo variable to remove the owner and the / from the string const repoName = repo.split('/')[1]; async function main() { - const PR = await (0, github_1.getSinglePR)(org, repoName, prNumber); - const assertion = await (0, github_1.getAssertion)(PR?.body ?? ''); - if (assertion?.length === 0 || assertion === null) { - console.log('No assertion found'); - core.setFailed('No assertion found'); - return; - } - else { + try { + const PR = await (0, github_1.getSinglePR)(org, repoName, prNumber); + // Try to use actual assertion, if not use description + let assertionToUse = await (0, github_1.getAssertion)(PR?.body ?? ''); + if (!assertionToUse) { + assertionToUse = (0, github_1.getDescription)(PR?.body ?? ''); + if (!assertionToUse) { + core.setFailed('No assertion or description found'); + return; + } + } const diff = await (0, github_1.getDiff)(prNumber, org, repoName); const changedFiles = (0, github_1.getChangedFiles)(diff); const file = await (0, github_1.getFileContent)(changedFiles, org, repoName); - const prompt = (0, open_ai_1.generatePrompt)(diff, assertion, file); + const prompt = (0, open_ai_1.generatePrompt)(diff, assertionToUse, file); const rawAnalysis = await (0, open_ai_1.testAssertion)(prompt); const analysis = (0, open_ai_1.writeAnalysis)(rawAnalysis); console.log(analysis); core.setOutput('analysis', analysis); - return analysis; + core.info('Analysis completed successfully'); + } + catch (error) { + core.setFailed(`Error during execution: ${error.message}`); } } main(); @@ -335,7 +357,7 @@ const testAssertion = async (prompt) => { ]; try { const chatCompletion = await openai.chat.completions.create({ - model: 'gpt-4-1106-preview', + model: 'gpt-4o', messages: conversation, response_format: { type: 'json_object' }, }); diff --git a/dist/open_ai/index.js b/dist/open_ai/index.js index f7990c2..20f49a5 100644 --- a/dist/open_ai/index.js +++ b/dist/open_ai/index.js @@ -99,7 +99,7 @@ const testAssertion = async (prompt) => { ]; try { const chatCompletion = await openai.chat.completions.create({ - model: 'gpt-4-1106-preview', + model: 'gpt-4o', messages: conversation, response_format: { type: 'json_object' }, }); From 9173b7aba4e25ce33ad159991baefcf11840091f Mon Sep 17 00:00:00 2001 From: Sean Park-Ross Date: Thu, 13 Jun 2024 17:49:48 +0100 Subject: [PATCH 7/7] Ignore .idea --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fd577f2..3f6e66f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ **.env node_modules -coverage \ No newline at end of file +coverage +.idea \ No newline at end of file