Skip to content

Commit c2bebc7

Browse files
xuwenyihustxuwenyihust
committed
Refactor SparkAppConfigModel, Notebook.js, and spark_app.py to update Spark app configuration handling
1 parent 2c2164f commit c2bebc7

File tree

5 files changed

+206
-105
lines changed

5 files changed

+206
-105
lines changed

‎server/app/routes/spark_app.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ def get_spark_app_config(notbook_path):
2121
def update_spark_app_config(notbook_path):
2222
logging.info(f"Updating spark app config for notebook path: {notbook_path}")
2323
data = request.get_json()
24-
return SparkApp.update_spark_app_config_by_notebook_id(notbook_path, data)
24+
return SparkApp.update_spark_app_config_by_notebook_path(notbook_path, data)

‎server/tests/routes/test_spark_app_route.py

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import unittest
2+
import json
3+
from flask_cors import CORS
4+
from flask import g
5+
from database import db
6+
from run import create_app
7+
from app.routes.notebook import notebook_blueprint
8+
from app.routes.login import login_blueprint
9+
from app.services.directory import Directory
10+
from app.models.user import UserModel
11+
from app.services.user import User
12+
from app.models.spark_app import SparkAppModel
13+
from app.models.notebook import NotebookModel
14+
15+
class SparkAppRouteTestCase(unittest.TestCase):
16+
17+
def setUp(self):
18+
self.app = create_app()
19+
self.app.register_blueprint(notebook_blueprint)
20+
self.app.register_blueprint(login_blueprint)
21+
self.client = self.app.test_client()
22+
with self.app.app_context():
23+
db.create_all()
24+
user = UserModel(name='test_user', email='test_email')
25+
user.set_password('test_password')
26+
db.session.add(user)
27+
db.session.commit()
28+
29+
def tearDown(self):
30+
with self.app.app_context():
31+
db.session.remove()
32+
db.drop_all()
33+
34+
def login_and_get_token(self):
35+
with self.app.app_context():
36+
response = self.client.post('/login', auth=('test_user', 'test_password'))
37+
return json.loads(response.data)['access_token']
38+
39+
def test_create_spark_app(self):
40+
with self.app.app_context():
41+
# Create Notebook
42+
notebook = NotebookModel(name='Test Notebook', path='/path/to/notebook', user_id=1)
43+
db.session.add(notebook)
44+
db.session.commit()
45+
46+
# Create Spark App
47+
spark_app_id = '1234'
48+
path = f'/spark-app/${spark_app_id}'
49+
50+
data = {
51+
'notebookPath': notebook.path
52+
}
53+
54+
token = self.login_and_get_token()
55+
headers = {
56+
'Authorization': f'Bearer {token}',
57+
}
58+
59+
response = self.client.get(
60+
path,
61+
headers=headers,
62+
data=json.dumps(data),
63+
)
64+
65+
self.assertEqual(response.status_code, 200)
66+
self.assertEqual(json.loads(response.data)['spark_app_id'], spark_app_id)
67+
self.assertEqual(json.loads(response.data)['notebook_id'], notebook.id)
68+
self.assertEqual(json.loads(response.data)['user_id'], notebook.user_id)
69+

‎webapp/src/components/notebook/Notebook.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ function Notebook({
324324
deleteNotebook={handleDeleteNotebook}
325325
/> : contentType === ContentType.Config ?
326326
<Config
327-
notebook={notebook}
327+
notebookPath={notebook.path}
328328
/> :
329329
<Runs
330330
notebook={notebook}

‎webapp/src/components/notebook/content/Config.js

Lines changed: 100 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,32 @@
11

22
import React, { useEffect, useState } from 'react';
33
import { Box, Card, CardHeader, CardContent, CardActions, Button, Typography, List, ListItem, ListItemText, TextField, Select, MenuItem } from '@mui/material';
4+
import SparkAppConfigModel from '../../../models/SparkAppConfigModel';
45

5-
function Config({ notebook }) {
6+
function Config({ notebookPath }) {
67

7-
const [executorCores, setExecutorCores] = useState('1');
8-
const [executorInstances, setExecutorInstances] = useState('1');
8+
const [loading, setLoading] = useState(true);
99

10-
const [executorMemory, setExecutorMemory] = useState('512');
10+
const [executorCores, setExecutorCores] = useState(null);
11+
const [executorInstances, setExecutorInstances] = useState(null);
12+
13+
const [executorMemory, setExecutorMemory] = useState(null);
1114
const [executorMemoryUnit, setExecutorMemoryUnit] = useState('m');
1215

1316
useEffect(() => {
14-
// const fetchSparkConfig = async () => {
17+
const fetchSparkConfig = async () => {
18+
const config = await SparkAppConfigModel.getSparkAppConfigByNotebookPath(notebookPath);
19+
console.log('config: ', config);
20+
21+
setExecutorCores(config.executor_cores);
22+
setExecutorInstances(config.executor_instances);
23+
setExecutorMemory(config.executor_memory);
24+
setExecutorMemoryUnit(config.executor_memory_unit);
1525

16-
// };
17-
// fetchSparkConfig();
18-
console.log('notebook:', notebook);
19-
}, [notebook]);
26+
setLoading(false);
27+
};
28+
fetchSparkConfig();
29+
}, [notebookPath]);
2030

2131
const handleExecutorMemoryUnitChange = (event) => {
2232
setExecutorMemoryUnit(event.target.value);
@@ -28,85 +38,89 @@ function Config({ notebook }) {
2838
console.log('executorMemory:', executorMemory + executorMemoryUnit);
2939
}
3040

31-
return (
32-
<Box sx={{
33-
marginTop: 5,
34-
marginRight: 5,
35-
marginLeft: 2,
36-
}}>
37-
<Card
38-
sx={{
39-
display: 'flex',
40-
flexDirection: 'column',
41-
}}>
42-
<CardHeader
43-
title={
44-
<Typography
45-
variant="body1"
46-
style={{ marginLeft: 10 }}
47-
color="textSecondary">
48-
Spark Configuration
49-
</Typography>
50-
}
51-
sx={{
52-
backgroundColor: '#f5f5f5',
53-
borderBottom: 1,
54-
borderBottomColor: '#f5f5f5',
55-
}}/>
56-
<CardContent>
57-
<List>
58-
<ListItem>
59-
<ListItemText primary="executor.memory" />
60-
<TextField
61-
defaultValue={executorMemory}
62-
variant="outlined"
63-
size="small"
64-
onInput={(e) => e.target.value = e.target.value.replace(/[^0-9]/g, '')}
65-
onChange={(e) => {
66-
setExecutorMemory(e.target.value)
67-
}}/>
68-
<Box m={1} />
69-
<Select value={executorMemoryUnit}
70-
size="small"
71-
onChange={handleExecutorMemoryUnitChange}>
72-
<MenuItem value={'m'}>MB</MenuItem>
73-
<MenuItem value={'g'}>GB</MenuItem>
74-
</Select>
75-
</ListItem>
41+
if (loading) {
42+
return <div>Loading...</div>;
43+
} else {
44+
return (
45+
<Box sx={{
46+
marginTop: 5,
47+
marginRight: 5,
48+
marginLeft: 2,
49+
}}>
50+
<Card
51+
sx={{
52+
display: 'flex',
53+
flexDirection: 'column',
54+
}}>
55+
<CardHeader
56+
title={
57+
<Typography
58+
variant="body1"
59+
style={{ marginLeft: 10 }}
60+
color="textSecondary">
61+
Spark Configuration
62+
</Typography>
63+
}
64+
sx={{
65+
backgroundColor: '#f5f5f5',
66+
borderBottom: 1,
67+
borderBottomColor: '#f5f5f5',
68+
}}/>
69+
<CardContent>
70+
<List>
71+
<ListItem>
72+
<ListItemText primary="spark.executor.memory" />
73+
<TextField
74+
defaultValue={executorMemory}
75+
variant="outlined"
76+
size="small"
77+
onInput={(e) => e.target.value = e.target.value.replace(/[^0-9]/g, '')}
78+
onChange={(e) => {
79+
setExecutorMemory(e.target.value)
80+
}}/>
81+
<Box m={1} />
82+
<Select value={executorMemoryUnit}
83+
size="small"
84+
onChange={handleExecutorMemoryUnitChange}>
85+
<MenuItem value={'m'}>MB</MenuItem>
86+
<MenuItem value={'g'}>GB</MenuItem>
87+
</Select>
88+
</ListItem>
7689

77-
<ListItem>
78-
<ListItemText primary="executor.cores" />
79-
<TextField
80-
defaultValue={executorCores}
81-
variant="outlined"
82-
size="small"
83-
onChange={(e) => setExecutorCores(e.target.value)} />
84-
</ListItem>
90+
<ListItem>
91+
<ListItemText primary="spark.executor.cores" />
92+
<TextField
93+
defaultValue={executorCores}
94+
variant="outlined"
95+
size="small"
96+
onChange={(e) => setExecutorCores(e.target.value)} />
97+
</ListItem>
8598

86-
<ListItem>
87-
<ListItemText primary="spark.executor.instances" />
88-
<TextField
89-
defaultValue={executorInstances}
90-
variant="outlined"
91-
size="small"
92-
onChange={(e) => setExecutorInstances(e.target.value)}/>
93-
</ListItem>
94-
</List>
95-
</CardContent>
96-
<CardActions>
97-
<Button
98-
variant="outlined"
99-
style={{
100-
marginLeft: '20px',
101-
borderColor: 'lightgrey',
102-
color: 'grey' }}
103-
onClick={handleSave}>
104-
Save
105-
</Button>
106-
</CardActions>
107-
</Card>
108-
</Box>
109-
);
99+
<ListItem>
100+
<ListItemText primary="spark.executor.instances" />
101+
<TextField
102+
defaultValue={executorInstances}
103+
variant="outlined"
104+
size="small"
105+
onChange={(e) => setExecutorInstances(e.target.value)}/>
106+
</ListItem>
107+
</List>
108+
</CardContent>
109+
<CardActions>
110+
<Button
111+
variant="outlined"
112+
style={{
113+
marginLeft: '20px',
114+
borderColor: 'lightgrey',
115+
color: 'grey' }}
116+
onClick={handleSave}>
117+
Save
118+
</Button>
119+
</CardActions>
120+
</Card>
121+
</Box>
122+
);
123+
}
110124
}
111125

112126
export default Config;

‎webapp/src/models/SparkAppConfigModel.js

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,45 @@ class SparkAppConfigModel {
44
constructor() {
55
}
66

7-
static async getSparkAppConfigByNotebookId() {
8-
const response = await fetch(`${config.serverBaseUrl}/spark_app/${}/config`);
7+
static async getSparkAppConfigByNotebookPath(notebookPath) {
8+
const response = await fetch(`${config.serverBaseUrl}/spark_app/${notebookPath}/config`);
99
if (!response.ok) {
1010
throw new Error('Failed to fetch Spark app config');
1111
} else {
1212
const data = await response.json();
13-
return data;
13+
console.log('Spark app config:', data);
14+
15+
const data_transformed = {};
16+
17+
const executorMemory = data['spark.executor.memory'];
18+
const memoryUnit = executorMemory.slice(-1);
19+
const memoryValue = executorMemory.slice(0, -1);
20+
data_transformed.executor_memory = memoryValue;
21+
data_transformed.executor_memory_unit = memoryUnit;
22+
23+
const executorCores = data['spark.executor.cores'];
24+
data_transformed.executor_cores = executorCores;
25+
26+
const executorInstances = data['spark.executor.instances'];
27+
data_transformed.executor_instances = executorInstances;
28+
29+
return data_transformed;
30+
}
31+
}
32+
33+
static async updateSparkAppConfig(notebookPath, sparkAppConfig) {
34+
const response = await fetch(`${config.serverBaseUrl}/spark_app/${notebookPath}/config`, {
35+
method: 'POST',
36+
headers: {
37+
'Content-Type': 'application/json',
38+
},
39+
body: JSON.stringify(sparkAppConfig),
40+
});
41+
42+
if (!response.ok) {
43+
throw new Error('Failed to update Spark app config');
1444
}
1545
}
46+
}
1647

17-
// static async updateSparkAppConfig(sparkAppConfig) {
18-
// const response = await fetch(`${config.serverBaseUrl}/spark_app_config`, {
19-
// method: 'PATCH',
20-
// headers: {
21-
// 'Content-Type': 'application/json',
22-
// },
23-
// body: JSON.stringify(sparkAppConfig),
24-
// });
25-
26-
// if (!response.ok) {
27-
// throw new Error('Failed to update Spark app config');
28-
// }
29-
// }
30-
}
48+
export default SparkAppConfigModel;

0 commit comments

Comments
 (0)