add buttons to rate logs

torchci/components/LogAnnotationToggle.tsx

@@ -0,0 +1,60 @@
+import React from "react";
+import { JobData, LogAnnotation } from "../lib/types";
+import { ToggleButtonGroup, ToggleButton } from "@mui/material";
+import { useSession } from "next-auth/react";
+
+export default function LogAnnotationToggle({
+  job,
+  annotation,
+  repo = null,
+  log_metadata,
+}: {
+  job: JobData;
+  annotation: LogAnnotation;
+  repo?: string | null;
+  log_metadata: Record<string, string>;
+}) {
+  const [state, setState] = React.useState<LogAnnotation>(
+    (annotation ?? "null") as LogAnnotation
+  );
+  const session = useSession();
+  async function handleChange(
+    _: React.MouseEvent<HTMLElement>,
+    newState: LogAnnotation
+  ) {
+    setState(newState);
+    const all_metadata = log_metadata;
+    all_metadata["job_id"] = job.id ?? "";
+    await fetch(`/api/log_annotation/${repo ?? job.repo}/${newState}`, {
+      method: "POST",
+      body: JSON.stringify(all_metadata),
+    });
+  }
+
+  return (
+    <>
+      Which log is preferable:{" "}
+      <ToggleButtonGroup
+        value={state}
+        exclusive
+        onChange={handleChange}
+        disabled={session.status !== "authenticated"}
+      >
+        {Object.keys(LogAnnotation).map((annotation, ind) => {
+          return (
+            <ToggleButton
+              key={ind}
+              value={annotation}
+              style={{ height: "12pt", textTransform: "none" }}
+            >
+              {
+                //@ts-ignore
+                LogAnnotation[annotation]
+              }
+            </ToggleButton>
+          );
+        })}
+      </ToggleButtonGroup>
+    </>
+  );
+}

torchci/components/LogViewer.tsx

@@ -17,9 +17,10 @@ import { parse } from "ansicolor";
 
 import { oneDark } from "@codemirror/theme-one-dark";
 import { isFailure } from "lib/JobClassifierUtil";
-import { JobData } from "lib/types";
+import { JobData, LogAnnotation } from "lib/types";
 import { useEffect, useRef, useState } from "react";
 import useSWRImmutable from "swr";
+import LogAnnotationToggle from "./LogAnnotationToggle";
 
 const ESC_CHAR_REGEX = /\x1b\[[0-9;]*m/g;
 // Based on the current editor view, produce a series of decorations that
@@ -190,7 +191,15 @@ function Log({ url, line }: { url: string; line: number | null }) {
   return <div ref={viewer}></div>;
 }
 
-export default function LogViewer({ job }: { job: JobData }) {
+export default function LogViewer({
+  job,
+  logRating = LogAnnotation.NULL,
+  showAnnotationToggle = false,
+}: {
+  job: JobData;
+  logRating?: LogAnnotation;
+  showAnnotationToggle?: boolean;
+}) {
   const [showLogViewer, setShowLogViewer] = useState(false);
 
   useEffect(() => {
@@ -222,6 +231,15 @@ export default function LogViewer({ job }: { job: JobData }) {
         <code>{job.failureLine ?? "Show log"}</code>
       </button>
       {showLogViewer && <Log url={job.logUrl!} line={job.failureLineNumber!} />}
+      {showAnnotationToggle && (
+        <div>
+          <LogAnnotationToggle
+            job={job}
+            log_metadata={{ job_id: "1" }}
+            annotation={logRating}
+          />
+        </div>
+      )}
     </div>
   );
 }

torchci/lib/types.ts

@@ -179,6 +179,14 @@ export enum JobAnnotation {
   OTHER = "Other",
 }
 
+export enum LogAnnotation {
+  NULL = "None",
+  PREFER_TOP_LOG = "Prefer Top Log",
+  PREFER_BOTTOM_LOG = "Prefer Bottom Log",
+  PREFER_NEITHER = "Prefer Neither",
+  SIMILAR_LOGS = "Similar Logs",
+}
+
 export function packHudParams(input: any) {
   return {
     repoOwner: input.repoOwner as string,

torchci/pages/api/log_annotation/[repoOwner]/[repoName]/[annotation].ts

@@ -0,0 +1,60 @@
+import { NextApiRequest, NextApiResponse } from "next";
+import { getDynamoClient } from "lib/dynamo";
+import { getServerSession } from "next-auth";
+import { authOptions } from "pages/api/auth/[...nextauth]";
+import { getOctokit } from "lib/github";
+import { hasWritePermissionsUsingOctokit } from "lib/bot/utils";
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  if (req.method !== "POST") {
+    return res.status(504).end();
+  }
+  // @ts-ignore
+  const session = await getServerSession(req, res, authOptions);
+  if (session === undefined || session === null || session.user === undefined) {
+    return res.status(401).end();
+  }
+
+  const { repoOwner, repoName, annotation } = req.query;
+  const repoOwnerStr = Array.isArray(repoOwner) ? repoOwner[0] : repoOwner;
+  const repoNameStr = Array.isArray(repoName) ? repoName[0] : repoName;
+  const octokit = await getOctokit(repoOwnerStr, repoNameStr);
+  const user = await octokit.rest.users.getAuthenticated();
+  const hasPermission = hasWritePermissionsUsingOctokit(
+    octokit,
+    user.data.login,
+    repoOwnerStr,
+    repoNameStr
+  );
+  if (!hasPermission) {
+    return res.status(401).end();
+  }
+  const log_metadata = JSON.parse(req.body) ?? [];
+  const client = getDynamoClient();
+  const jobId = log_metadata[0].job_id;
+  const dynamoKey = `${repoOwner}/${repoName}/${jobId}`;
+
+  const item: any = {
+    dynamoKey,
+    repo: `${repoOwner}/${repoName}`,
+    jobID: parseInt(jobId as string),
+  };
+
+  // TODO: we encode annotations as a string, but probably we want to just
+  // serialize a JSON object instead to avoid this silly special case.
+  if (annotation !== "null") {
+    item["annotationDecision"] = annotation;
+    item["annotationTime"] = new Date().toISOString();
+    item["annotationAuthor"] = user.data.login;
+    item["annotationLogMetadata"] = log_metadata;
+    item["metricType"] = "log_annotation";
+  }
+
+  return client.put({
+    TableName: "torchci-job-annotation",
+    Item: item,
+  });
+}