Add Docker build and push workflows for History Server, Notebook, and…

… Spark
xuwenyihust · Apr 27, 2024 · 3637b5b · 3637b5b
1 parent dab3898
commit 3637b5b
Show file tree

Hide file tree

Showing 11 changed files with 85 additions and 36 deletions.
diff --git a/.github/workflows/build-docker-history-server.yml b/.github/workflows/build-docker-history-server.yml
@@ -0,0 +1,27 @@
+name: Build Docker - History Server
+
+on:
+  push:
+    paths:
+      - 'docker/history-server/**'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Docker Build & Push
+        run: |
+          timestamp=$(date +"%Y%m%d%H%M%S")
+          
+          docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/history-server:$timestamp -f docker/history-server/Dockerfile ./docker/history-server
+          docker tag ${{ secrets.DOCKERHUB_USERNAME }}/history-server:$timestamp ${{ secrets.DOCKERHUB_USERNAME }}/history-server:latest
+
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/history-server:$timestamp
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/history-server:latest
diff --git a/.github/workflows/build-docker-notebook.yml b/.github/workflows/build-docker-notebook.yml
@@ -0,0 +1,27 @@
+name: Build Docker - Notebook
+
+on:
+  push:
+    paths:
+      - 'docker/notebook/**'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Docker Build & Push
+        run: |
+          timestamp=$(date +"%Y%m%d%H%M%S")
+          
+          docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/notebook:$timestamp -f docker/notebook/Dockerfile ./docker/notebook 
+          docker tag ${{ secrets.DOCKERHUB_USERNAME }}/notebook:$timestamp ${{ secrets.DOCKERHUB_USERNAME }}/notebook:latest
+
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/notebook:$timestamp
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/notebook:latest
diff --git a/.github/workflows/build-docker-spark.yml b/.github/workflows/build-docker-spark.yml
@@ -0,0 +1,28 @@
+name: Build Docker - Spark
+
+on:
+  push:
+    paths:
+      - 'docker/spark/**'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Docker Build & Push
+        run: |
+          timestamp=$(date +"%Y%m%d%H%M%S")
+          
+          docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/spark:$timestamp -f docker/spark/Dockerfile ./docker/spark
+          docker tag ${{ secrets.DOCKERHUB_USERNAME }}/spark:$timestamp ${{ secrets.DOCKERHUB_USERNAME }}/spark:latest
+
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/spark:$timestamp
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/spark:latest
+
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,6 +1,6 @@
 services:
-  jupyter:
-    image: wenyixu101/all-spark-notebook:latest
+  notebook:
+    image: wenyixu101/notebook:latest
     container_name: notebook
     ports:
       - "8888:8888"
@@ -16,7 +16,7 @@ services:
           memory: 1g
 
   spark-master:
-    image: wenyixu101/spark:3.5.0-python3.11
+    image: wenyixu101/spark:latest
     container_name: spark-master
     ports:
       - "4040:4040"

diff --git a/docker/spark-history-server/Dockerfile → docker/history-server/Dockerfile b/docker/spark-history-server/Dockerfile → docker/history-server/Dockerfile
diff --git a/docker/jupyter-notebook/Dockerfile.notebook → docker/notebook/Dockerfile b/docker/jupyter-notebook/Dockerfile.notebook → docker/notebook/Dockerfile
diff --git a/docker/jupyter-notebook/gcs_save_hook.py → docker/notebook/gcs_save_hook.py b/docker/jupyter-notebook/gcs_save_hook.py → docker/notebook/gcs_save_hook.py
diff --git a/...pyter-notebook/jupyter_notebook_config.py → docker/notebook/jupyter_notebook_config.py b/...pyter-notebook/jupyter_notebook_config.py → docker/notebook/jupyter_notebook_config.py
diff --git a/docker/jupyter-notebook/startup.py → docker/notebook/startup.py b/docker/jupyter-notebook/startup.py → docker/notebook/startup.py
diff --git a/docker/jupyter-notebook/Dockerfile.pyspark → docker/spark/Dockerfile b/docker/jupyter-notebook/Dockerfile.pyspark → docker/spark/Dockerfile