Skip to content

Commit

Permalink
upload train results to s3, add endpoint to get train results, fronte…
Browse files Browse the repository at this point in the history
…nd request from endpoint
  • Loading branch information
andrewpeng02 committed Apr 9, 2024
1 parent 8f9a616 commit 512024e
Show file tree
Hide file tree
Showing 28 changed files with 912 additions and 356 deletions.
15 changes: 15 additions & 0 deletions dlp-terraform/ecs/s3.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
resource "aws_s3_bucket" "s3bucket_executions" {
bucket = "dlp-executions"

tags = {
Name = "Execution data"
}
}
resource "aws_s3_bucket_public_access_block" "access_block_uploads" {
bucket = aws_s3_bucket.s3bucket_executions.id

block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}
9 changes: 9 additions & 0 deletions dlp-terraform/ecs/sqs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
resource "aws_sqs_queue" "training_queue" {
name = "training-queue.fifo"
fifo_queue = true
message_retention_seconds = 60*24
}

output "sqs_queue_url" {
value = aws_sqs_queue.training_queue.url
}
8 changes: 8 additions & 0 deletions frontend/src/features/Train/redux/trainspaceApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { backendApi } from "@/common/redux/backendApi";
import {
DATA_SOURCE,
DatasetData,
DetailedTrainResultsData,
FileUploadData,
} from "@/features/Train/types/trainTypes";
import { fetchBaseQuery } from "@reduxjs/toolkit/dist/query";
Expand All @@ -10,6 +11,12 @@ const trainspaceApi = backendApi
.enhanceEndpoints({ addTagTypes: ["UserDatasetFilesData"] })
.injectEndpoints({
endpoints: (builder) => ({
getTrainResultsData: builder.query<DetailedTrainResultsData, { trainspaceId: string }>({
query: ( {trainspaceId}) => ({
url: `/api/training/training/results/${trainspaceId}`
})
}
),
getDatasetFilesData: builder.query<
FileUploadData[],
{ dataSource: DATA_SOURCE }
Expand Down Expand Up @@ -95,6 +102,7 @@ const trainspaceApi = backendApi
});

export const {
useGetTrainResultsDataQuery,
useGetDatasetFilesDataQuery,
useUploadDatasetFileMutation,
useLazyGetColumnsFromDatasetQuery,
Expand Down
49 changes: 48 additions & 1 deletion frontend/src/features/Train/types/trainTypes.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { DATA_SOURCE_ARR } from "../constants/trainConstants";

// keep in sync with worker.py
export type DATA_SOURCE = typeof DATA_SOURCE_ARR[number];

export type TRAIN_STATUS =
Expand All @@ -16,16 +17,62 @@ export interface BaseTrainspaceData {
step: number;
}

// basic information, used on dashboard
export interface TrainResultsData {
name: string;
trainspaceId: number;
trainspaceId: string;
dataSource: DATA_SOURCE;
status: TRAIN_STATUS;
created: Date;
step: string;
uid: string;
}

export type CHART_TYPE = "LINE" | "AUC/ROC" | "CONFUSION_MATRIX"

export type Chart = TimeSeriesChart | AucRocChart | ConfusionMatrixChart

export interface TimeSeriesMetric {
x_name: string;
y_name: string;

x_values: number[];
y_values: number[];
}

export interface TimeSeriesChart {
name: string;

time_series: TimeSeriesMetric[]
chart_type: "LINE"
graph_index: number;
}

export interface AucRocChart {
name: string;

values: [number[], number[], number][];

chart_type: "AUC/ROC"
graph_index: number;
}

export interface ConfusionMatrixChart {
name: string;

values: number[][];

chart_type: "CONFUSION_MATRIX"
graph_index: number;
}

// more detailed information, used when viewing a run
export interface DetailedTrainResultsData {
basicInfo: TrainResultsData

allMetrics: Chart[]
}

export interface FileUploadData {
name: string;
lastModified: string;
Expand Down
Loading

0 comments on commit 512024e

Please sign in to comment.