Update notebook paths, dependencies, and Spark configuration in sg_re…

…sale_flat_prices.py, Dockerfile, and startup.py
xuwenyihust · May 8, 2024 · 61bfdc9 · 61bfdc9
1 parent 5279640
commit 61bfdc9
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 58 deletions.
diff --git a/.gitignore b/.gitignore
@@ -179,4 +179,6 @@ data/*
 */.jupyter/*
 */.local/*
 */.npm/*
-*/.custom_ipython_profile/*
+*/.custom_ipython_profile/*
+
+*/output/*
diff --git a/dags/sg_resale_flat_prices.py b/dags/sg_resale_flat_prices.py
@@ -24,7 +24,10 @@
     run_notebook = PapermillOperator(
         task_id='sg_resale_flat_prices_notebook',
         input_nb='/opt/airflow/examples/sg-resale-flat-prices/sg-resale-flat-prices.ipynb',
-        output_nb='/opt/airflow/examples/sg-resale-flat-prices/output/output-notebook-{{ execution_date }}.ipynb'
+        output_nb='/opt/airflow/examples/sg-resale-flat-prices/output/output-notebook-{{ execution_date }}.ipynb',
+        parameters={
+          'spark_master': 'spark://spark-master:7077'
+        },
     )
 
 run_notebook
diff --git a/docker/airflow/Dockerfile b/docker/airflow/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update
 USER airflow
 
 RUN pip install --upgrade pip && \
-    pip install apache-airflow-providers-papermill ipython jupyter ipykernel papermill pandas numpy matplotlib seaborn
+    pip install apache-airflow-providers-papermill ipython jupyter ipykernel papermill pandas numpy matplotlib seaborn pyspark
 
 # Add and install the Python 3 kernel
 RUN python3 -m ipykernel install --user --name python3 --display-name "Python 3"
diff --git a/examples/sg-resale-flat-prices/sg-resale-flat-prices.ipynb b/examples/sg-resale-flat-prices/sg-resale-flat-prices.ipynb