Add Docker build and push workflows, and update README and deployment…

… configurations

.github/workflows/build-docker.yml

@@ -0,0 +1,36 @@
+name: Build Docker
+
+on:
+  push:
+    branches:
+      - master  
+  pull_request:
+    paths:
+      - 'docker/**'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Docker Build & Push
+        run: |
+          timestamp=$(date +"%Y%m%d%H%M%S")
+          
+          docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/all-spark-notebook:$timestamp -f docker/jupyter-notebook/Dockerfile.notebook ./docker/jupyter-notebook 
+          docker tag ${{ secrets.DOCKERHUB_USERNAME }}/all-spark-notebook:$timestamp ${{ secrets.DOCKERHUB_USERNAME }}/all-spark-notebook:latest
+
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/all-spark-notebook:$timestamp
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/all-spark-notebook:latest
+
+          docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/spark-history-server:$timestamp -f docker/spark-history-server/Dockerfile ./docker/spark-history-server
+          docker tag ${{ secrets.DOCKERHUB_USERNAME }}/spark-history-server:$timestamp ${{ secrets.DOCKERHUB_USERNAME }}/spark-history-server:latest
+
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/spark-history-server:$timestamp
+          docker push ${{ secrets.DOCKERHUB_USERNAME }}/spark-history-server:latest

.github/workflows/build-deploy-examples.yml → .github/workflows/build-examples.yml

@@ -1,4 +1,4 @@
-name: Build & Deploy Examples
+name: Build Examples
 
 # Controls when the workflow will run
 on:
@@ -54,13 +54,4 @@ jobs:
           docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/word-count:$VERSION_WITH_TIMESTAMP --build-arg VERSION=$VERSION .
           docker push ${{ secrets.DOCKERHUB_USERNAME }}/word-count:$VERSION_WITH_TIMESTAMP
 
-  deploy-examples:
-    needs: build-examples
-    runs-on: ubuntu-latest
-    if: ${{ github.event.inputs.deploy-example != 'None' }}
-
-    steps:
-      - name: Deploy to GKE
-        run: |
-          echo "Deploy ${{ github.event.inputs.deploy-example }} to GKE!"
       

README.md

@@ -7,9 +7,21 @@
         <img alt="GitHub License" src="https://img.shields.io/github/license/xuwenyihust/Data-Platform?link=https%3A%2F%2Fgithub.com%2Fxuwenyihust%2FData-Platform%2Fblob%2Fmain%2FLICENSE">
 </p>
 
+> ⚠️ Currently in Development: Expect breaking changes and bugs!
+
 ## Overview
 A big data platform for data processing and machine learning based on Kubernetes and Spark.
 
+## Features
+- Spark Application Deployment
+    - Jar Application Submission
+    - PySpark Application Submission
+    - Jupyter Notebook
+        - Customized Integration with PySpark
+- Monitoring
+  - Spark UI
+  - History Server
+
 ## Supported Versions
 - Spark: 3.5.0
 - Scala: 2.12

helm/data-platform/templates/notebook-deployment.yaml

@@ -15,7 +15,7 @@ spec:
       serviceAccountName: spark
       containers:
       - name: notebook
-        image: wenyixu101/all-spark-notebook:spark-3.5.0_26
+        image: wenyixu101/all-spark-notebook:latest
         imagePullPolicy: Always
         command: ["/bin/bash", "-c", "start-notebook.sh"]
         ports:

helm/data-platform/templates/spark-history-server-deployment.yaml

@@ -14,7 +14,7 @@ spec:
     spec:
       containers:
       - name: spark-history-server
-        image: wenyixu101/spark-history-server:3.5.0
+        image: wenyixu101/spark-history-server:latest
         command: ["/bin/sh", "-c"]
         args: ["/opt/spark/bin/spark-class org.apache.spark.deploy.history.HistoryServer"]
         env: