Dataset test UI improvements

clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts

@@ -559,6 +559,20 @@ export const privacyRequestApi = baseApi.injectEndpoints({
         url: `privacy-request/${privacy_request_id}/filtered-results`,
       }),
     }),
+    getTestLogs: build.query<
+      Array<{
+        timestamp: string;
+        level: string;
+        module_info: string;
+        message: string;
+      }>,
+      { privacy_request_id: string }
+    >({
+      query: ({ privacy_request_id }) => ({
+        method: "GET",
+        url: `privacy-request/${privacy_request_id}/logs`,
+      }),
+    }),
   }),
 });
 
@@ -590,6 +604,7 @@ export const {
   useCreateTestConnectionMessageMutation,
   useGetPrivacyRequestAccessResultsQuery,
   useGetFilteredResultsQuery,
+  useGetTestLogsQuery,
 } = privacyRequestApi;
 
 export type CORSOrigins = Pick<SecurityApplicationConfig, "cors_origins">;

clients/admin-ui/src/features/test-datasets/TestLogsSection.tsx

@@ -0,0 +1,158 @@
+import { format } from "date-fns-tz";
+import { Box, Heading, HStack, Text } from "fidesui";
+import { memo, useCallback, useEffect, useMemo, useRef } from "react";
+import { useSelector } from "react-redux";
+
+import { useAppDispatch } from "~/app/hooks";
+import ClipboardButton from "~/features/common/ClipboardButton";
+import { useGetTestLogsQuery } from "~/features/privacy-requests";
+
+import {
+  selectLogs,
+  selectPrivacyRequestId,
+  setLogs,
+} from "./dataset-test.slice";
+
+const formatTimestamp = (isoTimestamp: string) => {
+  const date = new Date(isoTimestamp);
+  return format(date, "yyyy-MM-dd HH:mm:ss.SSS");
+};
+
+const getLevelColor = (level: string) => {
+  switch (level) {
+    case "ERROR":
+      return "red.500";
+    case "WARNING":
+      return "orange.500";
+    case "INFO":
+      return "blue.500";
+    default:
+      return "gray.500";
+  }
+};
+
+interface LogLineProps {
+  log: {
+    timestamp: string;
+    level: string;
+    module_info: string;
+    message: string;
+  };
+}
+
+const LogLine = memo(({ log }: LogLineProps) => (
+  <Box
+    as="pre"
+    margin={0}
+    fontSize="xs"
+    fontFamily="monospace"
+    whiteSpace="pre-wrap"
+    wordBreak="break-word"
+  >
+    <Text as="span" color="green.500">
+      {formatTimestamp(log.timestamp)}
+    </Text>
+    <Text as="span"> | </Text>
+    <Text as="span" color={getLevelColor(log.level)}>
+      {log.level.padEnd(8)}
+    </Text>
+    <Text as="span"> | </Text>
+    <Text as="span" color="cyan.500">
+      {log.module_info}
+    </Text>
+    <Text as="span"> - </Text>
+    <Text
+      as="span"
+      color={
+        log.level === "ERROR" || log.level === "WARNING"
+          ? getLevelColor(log.level)
+          : "gray.800"
+      }
+    >
+      {log.message}
+    </Text>
+  </Box>
+));
+
+LogLine.displayName = "LogLine";
+
+const TestLogsSection = () => {
+  const dispatch = useAppDispatch();
+  const logsRef = useRef<HTMLDivElement>(null);
+  const privacyRequestId = useSelector(selectPrivacyRequestId);
+  const logs = useSelector(selectLogs);
+
+  // Poll for logs when we have a privacy request ID
+  const { data: newLogs } = useGetTestLogsQuery(
+    { privacy_request_id: privacyRequestId! },
+    {
+      skip: !privacyRequestId,
+      pollingInterval: 1000,
+    },
+  );
+
+  // Update logs in store when new logs arrive
+  useEffect(() => {
+    if (newLogs) {
+      dispatch(setLogs(newLogs));
+    }
+  }, [newLogs, dispatch]);
+
+  // Auto scroll to bottom when new logs arrive
+  const scrollToBottom = useCallback(() => {
+    if (logsRef.current) {
+      logsRef.current.scrollTop = logsRef.current.scrollHeight;
+    }
+  }, []);
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [logs, scrollToBottom]);
+
+  // Format logs for copying to clipboard
+  const plainLogs = useMemo(
+    () =>
+      logs
+        ?.map(
+          (log) =>
+            `${formatTimestamp(log.timestamp)} | ${log.level} | ${log.module_info} - ${log.message}`,
+        )
+        .join("\n") || "",
+    [logs],
+  );
+
+  return (
+    <>
+      <Heading
+        as="h3"
+        size="sm"
+        display="flex"
+        alignItems="center"
+        justifyContent="space-between"
+      >
+        <HStack>
+          <Text>Test logs</Text>
+          <ClipboardButton copyText={plainLogs} />
+        </HStack>
+      </Heading>
+      <Box
+        ref={logsRef}
+        height="200px"
+        overflowY="auto"
+        borderWidth={1}
+        borderColor="gray.200"
+        borderRadius="md"
+        p={2}
+      >
+        {logs?.map((log) => (
+          <LogLine
+            key={`${log.timestamp}-${log.module_info}-${log.message}`}
+            log={log}
+          />
+        ))}
+      </Box>
+    </>
+  );
+};
+
+export default memo(TestLogsSection);

clients/admin-ui/src/features/test-datasets/TestRunnerSection.tsx

@@ -20,11 +20,14 @@ import {
   useGetDatasetInputsQuery,
   useTestDatastoreConnectionDatasetsMutation,
 } from "~/features/datastore-connections";
-import { useGetFilteredResultsQuery } from "~/features/privacy-requests";
+import { useGetPoliciesQuery } from "~/features/policy/policy.slice";
+import {
+  useGetFilteredResultsQuery,
+  useGetTestLogsQuery,
+} from "~/features/privacy-requests";
 import { PrivacyRequestStatus } from "~/types/api";
 import { isErrorResult } from "~/types/errors";
 
-import { useGetPoliciesQuery } from "../policy/policy.slice";
 import {
   finishTest,
   selectCurrentDataset,
@@ -67,12 +70,19 @@ const TestResultsSection = ({ connectionKey }: TestResultsSectionProps) => {
   }, [currentDataset, testInputs]);
 
   // Poll for results when we have a privacy request ID
-  const { data: filteredResults } = useGetFilteredResultsQuery(
+  const { data: filteredResults, error: filteredResultsError } =
+    useGetFilteredResultsQuery(
+      { privacy_request_id: privacyRequestId! },
+      {
+        skip: !privacyRequestId || !currentDataset?.fides_key,
+        pollingInterval: 2000,
+      },
+    );
+
+  // Get test logs with refetch capability
+  const { refetch: refetchLogs } = useGetTestLogsQuery(
     { privacy_request_id: privacyRequestId! },
-    {
-      skip: !privacyRequestId || !currentDataset?.fides_key,
-      pollingInterval: 2000,
-    },
+    { skip: !privacyRequestId },
   );
 
   // Get dataset inputs
@@ -124,6 +134,17 @@ const TestResultsSection = ({ connectionKey }: TestResultsSectionProps) => {
   useEffect(() => {
     const currentDatasetKey = currentDataset?.fides_key;
 
+    // Handle 404 errors by stopping polling and showing error
+    if (
+      filteredResultsError &&
+      "status" in filteredResultsError &&
+      filteredResultsError.status === 404
+    ) {
+      dispatch(finishTest());
+      toast(errorToastParams("Test run failed"));
+      return;
+    }
+
     if (
       !filteredResults ||
       filteredResults.privacy_request_id !== privacyRequestId ||
@@ -140,22 +161,30 @@ const TestResultsSection = ({ connectionKey }: TestResultsSectionProps) => {
 
     if (filteredResults.status === PrivacyRequestStatus.COMPLETE) {
       if (isTestRunning) {
-        dispatch(setTestResults(resultsAction));
-        dispatch(finishTest());
-        toast(successToastParams("Test run completed successfully"));
+        // Do one final log fetch before finishing
+        refetchLogs().then(() => {
+          dispatch(setTestResults(resultsAction));
+          dispatch(finishTest());
+          toast(successToastParams("Test run completed successfully"));
+        });
       }
     } else if (filteredResults.status === PrivacyRequestStatus.ERROR) {
-      dispatch(setTestResults(resultsAction));
-      dispatch(finishTest());
-      toast(errorToastParams("Test run failed"));
+      // Do one final log fetch before finishing
+      refetchLogs().then(() => {
+        dispatch(setTestResults(resultsAction));
+        dispatch(finishTest());
+        toast(errorToastParams("Test run failed"));
+      });
     }
   }, [
     filteredResults,
+    filteredResultsError,
     privacyRequestId,
     currentDataset,
     isTestRunning,
     dispatch,
     toast,
+    refetchLogs,
   ]);
 
   const handleInputChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {

clients/admin-ui/src/features/test-datasets/dataset-test.slice.ts

@@ -11,6 +11,12 @@ interface DatasetTestState {
   testResults: Record<string, string>;
   isTestRunning: boolean;
   currentPolicyKey?: string;
+  logs: Array<{
+    timestamp: string;
+    level: string;
+    module_info: string;
+    message: string;
+  }>;
 }
 
 const initialState: DatasetTestState = {
@@ -20,6 +26,7 @@ const initialState: DatasetTestState = {
   testInputs: {},
   testResults: {},
   isTestRunning: false,
+  logs: [],
 };
 
 export const datasetTestSlice = createSlice({
@@ -32,6 +39,7 @@ export const datasetTestSlice = createSlice({
         [action.payload]: "",
       };
       draftState.isTestRunning = true;
+      draftState.logs = [];
     },
     setPrivacyRequestId: (draftState, action: PayloadAction<string>) => {
       draftState.privacyRequestId = action.payload;
@@ -104,6 +112,12 @@ export const datasetTestSlice = createSlice({
         [action.payload.datasetKey]: action.payload.values,
       };
     },
+    setLogs: (draftState, action: PayloadAction<typeof initialState.logs>) => {
+      draftState.logs = action.payload;
+    },
+    clearLogs: (draftState) => {
+      draftState.logs = [];
+    },
   },
 });
 
@@ -116,6 +130,8 @@ export const {
   setCurrentDataset,
   setReachability,
   setTestResults,
+  setLogs,
+  clearLogs,
 } = datasetTestSlice.actions;
 
 export const selectPrivacyRequestId = (state: RootState) =>
@@ -142,5 +158,6 @@ export const selectTestResults = (state: RootState) => {
 };
 export const selectIsTestRunning = (state: RootState) =>
   state.datasetTest.isTestRunning;
+export const selectLogs = (state: RootState) => state.datasetTest.logs;
 
 export const { reducer } = datasetTestSlice;

clients/admin-ui/src/pages/systems/configure/[id]/test-datasets.tsx

@@ -1,4 +1,4 @@
-import { HStack } from "fidesui";
+import { Box, HStack, VStack } from "fidesui";
 import type { NextPage } from "next";
 import { useRouter } from "next/router";
 import { useEffect } from "react";
@@ -13,6 +13,7 @@ import {
   useGetSystemByFidesKeyQuery,
 } from "~/features/system";
 import EditorSection from "~/features/test-datasets/DatasetEditorSection";
+import TestLogsSection from "~/features/test-datasets/TestLogsSection";
 import TestResultsSection from "~/features/test-datasets/TestRunnerSection";
 
 // Helper functions
@@ -67,16 +68,27 @@ const TestDatasetPage: NextPage = () => {
           { title: "Test datasets" },
         ]}
       />
-      <HStack
+      <VStack
         alignItems="stretch"
         flex="1"
         minHeight="0"
         spacing="4"
-        padding="4 4 4 0"
+        padding="0"
       >
-        <EditorSection connectionKey={connectionKey} />
-        <TestResultsSection connectionKey={connectionKey} />
-      </HStack>
+        <HStack
+          alignItems="stretch"
+          flex="1"
+          minHeight="0"
+          spacing="4"
+          maxHeight="60vh"
+        >
+          <EditorSection connectionKey={connectionKey} />
+          <TestResultsSection connectionKey={connectionKey} />
+        </HStack>
+        <Box flex="0 0 auto">
+          <TestLogsSection />
+        </Box>
+      </VStack>
     </Layout>
   );
 };