From 6feef348f03ea68f375a6ea015ac778e043730c9 Mon Sep 17 00:00:00 2001
From: FalloutFalcon <86381784+FalloutFalcon@users.noreply.github.com>
Date: Mon, 28 Oct 2024 15:02:44 -0500
Subject: [PATCH] Flaky comments (#3639)

<!-- Write **BELOW** The Headers and **ABOVE** The comments else it may
not be viewable. -->
<!-- You can view Contributing.MD for a detailed description of the pull
request process. -->

## About The Pull Request
Flaky tests can now comment under issues it created to document further
failures, allowing us to track how often our flaky tests are flaking
Also I ran prettier.
<!-- Describe The Pull Request. Please be sure every change is
documented or this can delay review and even discourage maintainers from
merging your PR! -->

## Why It's Good For The Game
This is useful to remind us that a flaky test is a reoccurring issue and
not a one of thing that's already been fixed.

<!-- Please add a short description of why you think these changes would
benefit the game. If you can't justify it in words, it might not be
worth adding. -->

## Changelog

:cl:
code: flaky tests can now comment under its own prs.
/:cl:

<!-- Both :cl:'s are required for the changelog to work! You can put
your name to the right of the first :cl: if you want to overwrite your
GitHub username as author ingame. -->
<!-- You can use multiple of the same prefix (they're only used for the
icon ingame) and delete the unneeded ones. Despite some of the tags,
changelogs should generally represent how a player might be affected by
the changes rather than a summary of the PR's contents. -->
---
 tools/pull_request_hooks/rerunFlakyTests.js | 72 ++++++++++++++++-----
 1 file changed, 55 insertions(+), 17 deletions(-)

diff --git a/tools/pull_request_hooks/rerunFlakyTests.js b/tools/pull_request_hooks/rerunFlakyTests.js
index d3085a67260b..41c997c0bef2 100644
--- a/tools/pull_request_hooks/rerunFlakyTests.js
+++ b/tools/pull_request_hooks/rerunFlakyTests.js
@@ -2,10 +2,8 @@ const LABEL = "🤖 Flaky Test Report";
 const TITLE_BOT_HEADER = "title: ";
 
 // Only check jobs that start with these.
-// Helps make sure we don't restart something like screenshot tests or linters, which are not known to be flaky.
-const CONSIDERED_JOBS = [
-	"Integration Tests",
-];
+// Helps make sure we don't restart something like linters, which are not known to be flaky.
+const CONSIDERED_JOBS = ["Integration Tests"];
 
 async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) {
 	const jobs = await github.paginate(
@@ -14,14 +12,14 @@ async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) {
 			owner: context.repo.owner,
 			repo: context.repo.repo,
 			run_id: workflowRunId,
-			attempt_number: runAttempt
+			attempt_number: runAttempt,
 		},
-		response => {
+		(response) => {
 			return response.data;
-		});
+		}
+	);
 
-	return jobs
-		.filter((job) => job.conclusion === "failure");
+	return jobs.filter((job) => job.conclusion === "failure");
 }
 
 export async function rerunFlakyTests({ github, context }) {
@@ -33,7 +31,7 @@ export async function rerunFlakyTests({ github, context }) {
 	);
 
 	const filteredFailingJobs = failingJobs.filter((job) => {
-		console.log(`Failing job: ${job.name}`)
+		console.log(`Failing job: ${job.name}`);
 		return CONSIDERED_JOBS.some((title) => job.name.startsWith(title));
 	});
 	if (filteredFailingJobs.length === 0) {
@@ -139,7 +137,9 @@ export function extractDetails(log) {
 	if (runtimeMatch) {
 		const runtime = runtimeMatch.groups.error.trim();
 
-		const invalidTimerMatch = runtime.match(/^Invalid timer:.+object:(?<object>[^[]+).*delegate:(?<proc>.+?), source:/);
+		const invalidTimerMatch = runtime.match(
+			/^Invalid timer:.+object:(?<object>[^[]+).*delegate:(?<proc>.+?), source:/
+		);
 		if (invalidTimerMatch) {
 			return {
 				title: `Flaky test ${failGroup}: Invalid timer: ${invalidTimerMatch.groups.proc.trim()} on ${invalidTimerMatch.groups.object.trim()}`,
@@ -153,7 +153,9 @@ export function extractDetails(log) {
 		};
 	}
 
-	const hardDelMatch = failure.headline.match(/^(?<object>\/[\w/]+) hard deleted .* times out of a total del count of/);
+	const hardDelMatch = failure.headline.match(
+		/^(?<object>\/[\w/]+) hard deleted .* times out of a total del count of/
+	);
 	if (hardDelMatch) {
 		return {
 			title: `Flaky hard delete: ${hardDelMatch.groups.object}`,
@@ -202,7 +204,9 @@ async function getExistingIssueId(graphql, context, title) {
 		}
 	);
 
-	const exactTitle = openFlakyTestIssues.find((issue) => issue.title === title);
+	const exactTitle = openFlakyTestIssues.find(
+		(issue) => issue.title === title
+	);
 	if (exactTitle !== undefined) {
 		return exactTitle.number;
 	}
@@ -229,7 +233,27 @@ function createBody({ title, failures }, runUrl) {
 	${failures
 		.map(
 			(failure) =>
-				`${failure.group}: ${failure.headline}\n\t${failure.details.join("\n")}`
+				`${failure.group}: ${
+					failure.headline
+				}\n\t${failure.details.join("\n")}`
+		)
+		.join("\n")}
+	\`\`\`
+	`.replace(/^\s*/gm, "");
+}
+
+function createComment(failures, runUrl) {
+	return `
+	Flaky tests were detected again in [this test run](${runUrl}).
+
+	Failures:
+	\`\`\`
+	${failures
+		.map(
+			(failure) =>
+				`${failure.group}: ${
+					failure.headline
+				}\n\t${failure.details.join("\n")}`
 		)
 		.join("\n")}
 	\`\`\`
@@ -245,7 +269,7 @@ export async function reportFlakyTests({ github, context }) {
 	);
 
 	const filteredFailingJobs = failedJobsFromLastRun.filter((job) => {
-		console.log(`Failing job: ${job.name}`)
+		console.log(`Failing job: ${job.name}`);
 		return CONSIDERED_JOBS.some((title) => job.name.startsWith(title));
 	});
 
@@ -275,8 +299,22 @@ export async function reportFlakyTests({ github, context }) {
 		);
 
 		if (existingIssueId !== undefined) {
-			// Maybe in the future, if it's helpful, update the existing issue with new links
-			console.log(`Existing issue found: #${existingIssueId}`);
+			await github.rest.issues.createComment({
+				owner: context.repo.owner,
+				repo: context.repo.repo,
+				issue_number: existingIssueId,
+				body: createComment(
+					details.failures,
+					`https://github.com/${context.repo.owner}/${
+						context.repo.repo
+					}/actions/runs/${
+						context.payload.workflow_run.id
+					}/attempts/${context.payload.workflow_run.run_attempt - 1}`
+				),
+			});
+			console.log(
+				`Existing issue found: #${existingIssueId}, updated it with a comment`
+			);
 			return;
 		}