Add resources limits for notebook and spark-master services

xuwenyihust · Feb 26, 2024 · e24e2dd · e24e2dd
1 parent c83eaba
commit e24e2dd
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 2 deletions.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -9,4 +9,29 @@ services:
       - ENVIRONMENT=development
     command: "start-notebook.sh --NotebookApp.token='' --NotebookApp.password=''"
     pull_policy: always
+    deploy:
+      resources:
+        limits:
+          cpus: "1" 
+          memory: 1g
+
+  spark-master:
+    image: wenyixu101/spark:3.5.0-python3.11
+    container_name: spark-master
+    ports:
+      - "4040:4040"
+      - "8080:8080"
+      - "7077:7077"
+    environment:
+      - SPARK_MASTER_HOST=spark-master
+      - SPARK_MODE=master
+      - SPARK_MASTER_PORT=7077
+      - SPARK_MASTER_WEBUI_PORT=8080
+    command: "/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master"
+    pull_policy: always
+    deploy:
+      resources:
+        limits:
+          cpus: "1" 
+          memory: 1g
 
diff --git a/docker/jupyter-notebook/startup.py b/docker/jupyter-notebook/startup.py
@@ -118,5 +118,5 @@ def create_spark_dev():
 # else:
 #     raise ValueError(f"Invalid environment: {environment}")
 
-spark = create_spark_dev()
-display_msg()
+# spark = create_spark_dev()
+# display_msg()