Skip to content

Commit 61944c9

Browse files
martinjrobinseatyourgreens
martinjrobins
authored and
eatyourgreens
committed
add data upload, map headers and set time unit
Fix type errors for data sub-page tabs Case insensitive header normalisation Trim blank lines from CSV Add Administration ID to the CSV data parser Allow time_unit as a header
1 parent 13aa587 commit 61944c9

14 files changed

+508
-3
lines changed

‎frontend-v2/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"postcss": "^8.4.32",
2424
"react": "^18.2.0",
2525
"react-dom": "^18.2.0",
26+
"react-dropzone": "^14.2.3",
2627
"react-hook-form": "^7.43.9",
2728
"react-player": "^2.13.0",
2829
"react-plotly.js": "^2.6.0",

‎frontend-v2/src/features/data/Data.tsx

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { DynamicTabs, TabPanel } from "../../components/DynamicTabs";
2+
import LoadDataTab from "./LoadDataTab";
3+
import { SubPageName } from "../main/mainSlice";
4+
5+
const Data: React.FC = () => {
6+
const tabKeys = [
7+
SubPageName.LOAD_DATA,
8+
SubPageName.STRATIFICATION,
9+
SubPageName.VISUALISATION
10+
];
11+
return (
12+
<DynamicTabs tabNames={tabKeys}>
13+
<TabPanel>
14+
<LoadDataTab />
15+
</TabPanel>
16+
<TabPanel>
17+
<LoadDataTab />
18+
</TabPanel>
19+
<TabPanel>
20+
<LoadDataTab />
21+
</TabPanel>
22+
</DynamicTabs>
23+
);
24+
}
25+
26+
export default Data;

‎frontend-v2/src/features/data/LoadData.tsx

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import { Alert, Box, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from '@mui/material';
2+
import Papa, { ParseError, ParseMeta } from 'papaparse'
3+
import React, {useCallback, useState} from 'react'
4+
import {useDropzone} from 'react-dropzone'
5+
import MapHeaders from './MapHeaders';
6+
import { manditoryHeaders, normaliseHeader, normalisedHeaders } from './normaliseDataHeaders';
7+
import { StepperState } from './LoadDataStepper';
8+
9+
export type Row = {[key: string]: string};
10+
export type Data = Row[];
11+
export type Field = string;
12+
13+
14+
const style = {
15+
dropArea: {
16+
width: "100%",
17+
height: "150px",
18+
border: "2px dashed #000",
19+
marginBottom: "10px",
20+
display: "flex",
21+
justifyContent: "center",
22+
alignItems: "center",
23+
cursor: "pointer",
24+
':hover': {
25+
backgroundColor: "#f0f0f0"
26+
}
27+
},
28+
dropAreaContainer: {
29+
display: "flex",
30+
justifyContent: "center",
31+
alignItems: "center",
32+
}
33+
};
34+
35+
interface ILoadDataProps {
36+
state: StepperState;
37+
firstTime: boolean;
38+
}
39+
40+
const validateNormalisedFields = (fields: Field[]) => {
41+
const errors: string[] = [];
42+
// check for mandatory fields
43+
for (const field of manditoryHeaders) {
44+
if (!fields.includes(field)) {
45+
errors.push(`${field} has not been defined`);
46+
}
47+
}
48+
return errors;
49+
}
50+
51+
const LoadData: React.FC<ILoadDataProps> = ({state, firstTime}) => {
52+
const [errors, setErrors] = useState<string[]>(firstTime ? [] : validateNormalisedFields(state.normalisedFields));
53+
const [showData, setShowData] = useState<boolean>(state.data.length > 0 && state.fields.length > 0);
54+
55+
const onDrop = useCallback((acceptedFiles: File[]) => {
56+
acceptedFiles.forEach((file) => {
57+
const reader = new FileReader()
58+
59+
reader.onabort = () => setErrors(['file reading was aborted'])
60+
reader.onerror = () => setErrors(['file reading has failed'])
61+
reader.onload = () => {
62+
// Parse the CSV data
63+
const rawCsv = reader.result as string;
64+
const csvData = Papa.parse(rawCsv.trim(), { header: true });
65+
const data = csvData.data as Data;
66+
const fields = csvData.meta.fields || [];
67+
const normalisedFields = fields.map(normaliseHeader);
68+
const errors = csvData.errors.map((e) => e.message).concat(validateNormalisedFields(normalisedFields));
69+
state.setData(csvData.data as Data);
70+
state.setFields(fields);
71+
state.setNormalisedFields(normalisedFields)
72+
setErrors(errors);
73+
if (csvData.data.length > 0 && csvData.meta.fields) {
74+
setShowData(true);
75+
}
76+
}
77+
reader.readAsText(file)
78+
})
79+
80+
}, [])
81+
const {getRootProps, getInputProps} = useDropzone({onDrop})
82+
83+
const setNormalisedFields = (fields: Field[]) => {
84+
state.setNormalisedFields(fields);
85+
setErrors(validateNormalisedFields(fields));
86+
}
87+
88+
89+
return (
90+
<Stack spacing={2}>
91+
<div style={style.dropAreaContainer}>
92+
<div {...getRootProps({style: style.dropArea})}>
93+
<input {...getInputProps()} />
94+
<p>Drag 'n' drop some files here, or click to select files</p>
95+
</div>
96+
</div>
97+
<Box sx={{ width: '100%', maxHeight: "20vh", overflow: 'auto', whiteSpace: 'nowrap'}}>
98+
{errors.map((error, index) => (
99+
<Alert severity="error" key={index}>{error}</Alert>
100+
))}
101+
</Box>
102+
<Box component="div" sx={{ maxHeight: "40vh", overflow: 'auto', overflowX: 'auto' }}>
103+
{showData && <MapHeaders data={state.data} fields={state.fields} setNormalisedFields={setNormalisedFields} normalisedFields={state.normalisedFields}/>}
104+
</Box>
105+
</Stack>
106+
)
107+
}
108+
109+
export default LoadData;

‎frontend-v2/src/features/data/LoadDataStepper.tsx

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import * as React from 'react';
2+
import Box from '@mui/material/Box';
3+
import Stepper from '@mui/material/Stepper';
4+
import Step from '@mui/material/Step';
5+
import StepLabel from '@mui/material/StepLabel';
6+
import Button from '@mui/material/Button';
7+
import Typography from '@mui/material/Typography';
8+
import LoadData from './LoadData';
9+
import { useState } from 'react';
10+
import MapObservations from './MapObservations';
11+
12+
const stepLabels = ['Upload Data', 'Map Observations'];
13+
const stepComponents = [LoadData, MapObservations];
14+
15+
type Row = {[key: string]: string};
16+
type Data = Row[];
17+
type Field = string;
18+
19+
export type StepperState = {
20+
fields: Field[];
21+
normalisedFields: Field[];
22+
data: Data;
23+
timeUnit?: string;
24+
setTimeUnit: (timeUnit: string) => void;
25+
setFields: (fields: Field[]) => void;
26+
setNormalisedFields: (fields: Field[]) => void;
27+
setData: (data: Data) => void;
28+
amountUnit?: string;
29+
setAmountUnit: (amountUnit: string) => void;
30+
observationUnits?: {[key: string]: string};
31+
setObservationUnits: (observationUnits: {[key: string]: string}) => void;
32+
}
33+
34+
const LoadDataStepper: React.FC = () => {
35+
const [data, setData] = useState<Data>([]);
36+
const [fields, setFields] = useState<string[]>([]);
37+
const [normalisedFields, setNormalisedFields] = useState<string[]>([]);
38+
const [timeUnit, setTimeUnit] = useState<string | undefined>(undefined);
39+
const [amountUnit, setAmountUnit] = useState<string | undefined>(undefined);
40+
const [observationUnits, setObservationUnits] = useState<{[key: string]: string}>({});
41+
42+
const state = {
43+
fields,
44+
normalisedFields,
45+
data,
46+
setFields,
47+
setNormalisedFields,
48+
setData,
49+
timeUnit,
50+
setTimeUnit,
51+
amountUnit,
52+
setAmountUnit,
53+
observationUnits,
54+
setObservationUnits,
55+
};
56+
57+
const [stepState, setStepState] = useState({ activeStep: 0, maxStep: 0 });
58+
const StepComponent = stepComponents[stepState.activeStep];
59+
60+
const handleNext = () => {
61+
setStepState((prevActiveStep) => ({
62+
activeStep: prevActiveStep.activeStep + 1,
63+
maxStep: Math.max(prevActiveStep.maxStep, prevActiveStep.activeStep + 1)
64+
}));
65+
};
66+
67+
const handleBack = () => {
68+
setStepState((prevActiveStep) => ({ ...prevActiveStep, activeStep: prevActiveStep.activeStep - 1 }));
69+
};
70+
71+
return (
72+
<Box sx={{ width: '100%' }}>
73+
<Stepper activeStep={stepState.activeStep} alternativeLabel>
74+
{stepLabels.map((step, index) => (
75+
<Step key={index}>
76+
<StepLabel>{step}</StepLabel>
77+
</Step>
78+
))}
79+
</Stepper>
80+
<Typography>{stepState.activeStep === stepLabels.length ? 'The process is completed' : <StepComponent state={state} firstTime={stepState.activeStep === stepState.maxStep}/>}</Typography>
81+
<Box sx={{ display: 'flex', justifyContent: 'space-between', marginTop: 1 }}>
82+
<Button disabled={stepState.activeStep === 0} onClick={handleBack}>Back</Button>
83+
<Button variant="contained" color="primary" onClick={handleNext}>
84+
{stepState.activeStep === stepLabels.length - 1 ? 'Finish' : 'Next'}
85+
</Button>
86+
</Box>
87+
</Box>
88+
);
89+
}
90+
91+
export default LoadDataStepper;

‎frontend-v2/src/features/data/LoadDataTab.tsx

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import * as React from 'react';
2+
import Button from '@mui/material/Button';
3+
import Avatar from '@mui/material/Avatar';
4+
import List from '@mui/material/List';
5+
import ListItem from '@mui/material/ListItem';
6+
import ListItemAvatar from '@mui/material/ListItemAvatar';
7+
import ListItemButton from '@mui/material/ListItemButton';
8+
import ListItemText from '@mui/material/ListItemText';
9+
import DialogTitle from '@mui/material/DialogTitle';
10+
import Dialog from '@mui/material/Dialog';
11+
import PersonIcon from '@mui/icons-material/Person';
12+
import AddIcon from '@mui/icons-material/Add';
13+
import Typography from '@mui/material/Typography';
14+
import { blue } from '@mui/material/colors';
15+
import LoadDataStepper from './LoadDataStepper';
16+
import { DialogContent } from '@mui/material';
17+
18+
export interface LoadDataDialogProps {
19+
open: boolean;
20+
onClose: () => void;
21+
}
22+
23+
function LoadDataDialog(props: LoadDataDialogProps) {
24+
const { onClose, open } = props;
25+
26+
const handleClose = () => {
27+
onClose();
28+
};
29+
30+
return (
31+
<Dialog onClose={handleClose} open={open} maxWidth='lg' fullWidth>
32+
<DialogTitle>Upload New Dataset</DialogTitle>
33+
<DialogContent>
34+
<LoadDataStepper />
35+
</DialogContent>
36+
</Dialog>
37+
);
38+
}
39+
40+
export default function LoadDataTab() {
41+
const [open, setOpen] = React.useState(false);
42+
43+
const handleClickOpen = () => {
44+
setOpen(true);
45+
};
46+
47+
const handleClose = () => {
48+
setOpen(false);
49+
};
50+
51+
return (
52+
<div>
53+
<Button variant="outlined" onClick={handleClickOpen}>
54+
Upload New Dataset
55+
</Button>
56+
<LoadDataDialog
57+
open={open}
58+
onClose={handleClose}
59+
/>
60+
</div>
61+
);
62+
}

‎frontend-v2/src/features/data/MapHeaders.tsx

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import { TableContainer, Table, TableHead, TableRow, TableCell, TableBody, Select, FormControl, MenuItem, InputLabel, Typography } from "@mui/material";
2+
import LoadData, { Data, Field } from "./LoadData";
3+
import { normalisedHeaders } from "./normaliseDataHeaders";
4+
5+
interface IMapHeaders {
6+
data: Data;
7+
fields: Field[];
8+
normalisedFields: Field[];
9+
setNormalisedFields: (fields: Field[]) => void;
10+
}
11+
12+
const MapHeaders: React.FC<IMapHeaders> = ({data, fields, normalisedFields, setNormalisedFields}: IMapHeaders) => {
13+
14+
const normalisedHeadersOptions = normalisedHeaders.map((header) => ({value: header, label: header}));
15+
16+
const handleFieldChange = (index: number) => (event: any) => {
17+
const newFields = [...normalisedFields];
18+
newFields[index] = event.target.value;
19+
setNormalisedFields(newFields);
20+
}
21+
22+
return (
23+
<Table>
24+
<TableHead>
25+
<TableRow>
26+
{fields.map((field, index) => (
27+
<TableCell key={index}>
28+
<Typography variant="h6" component="div" sx={{ flexGrow: 1, marginBottom: 1 }} align="center">
29+
{field}
30+
</Typography>
31+
<FormControl fullWidth>
32+
<InputLabel id={`select-${index}-label`}>Column Type</InputLabel>
33+
<Select
34+
labelId={`select-${index}-label`}
35+
id={`select-${index}`}
36+
value={normalisedFields[index]}
37+
label="Column Type"
38+
onChange={handleFieldChange(index)}
39+
>
40+
{normalisedHeadersOptions.map((option) => (
41+
<MenuItem key={option.value} value={option.value}>{option.label}</MenuItem>
42+
))}
43+
</Select>
44+
</FormControl>
45+
</TableCell>
46+
))}
47+
</TableRow>
48+
</TableHead>
49+
<TableBody>
50+
{data.map((row, index) => (
51+
<TableRow key={index}>
52+
{fields.map((field, index) => (
53+
<TableCell key={index}>{row[field]}</TableCell>
54+
))}
55+
</TableRow>
56+
))}
57+
</TableBody>
58+
</Table>
59+
)
60+
}
61+
62+
export default MapHeaders;

‎frontend-v2/src/features/data/MapObservations.tsx

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { Field, Data } from "./LoadData";
2+
import { StepperState } from "./LoadDataStepper";
3+
4+
interface IMapObservations {
5+
state: StepperState;
6+
firstTime: boolean;
7+
}
8+
9+
const MapObservations: React.FC<IMapObservations> = ({state, firstTime}: IMapObservations) => {
10+
return (null)
11+
}
12+
13+
export default MapObservations;
14+
15+

0 commit comments

Comments
 (0)