From 521a4dd4b1a29e1b098592a9e4d9e319a4275a06 Mon Sep 17 00:00:00 2001
From: Luis Belloch <luis@luisbelloch.es>
Date: Tue, 8 Dec 2020 22:49:17 +0100
Subject: [PATCH] Update Spark to 3.0.1 version (Java 11, Python 3.9)

---
 .envrc                  | 4 ++--
 infra/docker/.envrc     | 2 ++
 infra/docker/Dockerfile | 6 +++---
 infra/docker/Makefile   | 4 ++--
 local_setup.sh          | 2 +-
 playbook.yml            | 8 ++++----
 6 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 infra/docker/.envrc

diff --git a/.envrc b/.envrc
index 22de281..bd1a7cf 100644
--- a/.envrc
+++ b/.envrc
@@ -1,5 +1,5 @@
-layout python3
-# use sdk java 8.0.232.hs-adpt
+layout pyenv 3.9.0
+use sdk java 11.0.9.hs-adpt
 export VBOXMANAGE=/Applications/VirtualBox.app/Contents/MacOS
 export SPARK_HOME=$(pwd)/.spark
 export PATH=$SPARK_HOME/bin:$VBOXMANAGE:$PATH
diff --git a/infra/docker/.envrc b/infra/docker/.envrc
new file mode 100644
index 0000000..df70d3d
--- /dev/null
+++ b/infra/docker/.envrc
@@ -0,0 +1,2 @@
+export DOCKER_BUILDKIT=1
+export COMPOSE_DOCKER_CLI_BUILD=1
diff --git a/infra/docker/Dockerfile b/infra/docker/Dockerfile
index 5e4f001..32d7bfb 100644
--- a/infra/docker/Dockerfile
+++ b/infra/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:8-jdk-slim
+FROM openjdk:11-jdk-slim
 LABEL maintainer="Luis Belloch <docker@luisbelloch.es>"
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -6,9 +6,9 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends python3-software-properties python3-numpy curl && \
     rm -rf /var/lib/apt/lists/*
 
-ARG SPARK_VERSION=2.4.5
+ARG SPARK_VERSION=3.0.1
 ENV SPARK_HOME=/opt/spark
-RUN mkdir -p /opt/spark && curl -s http://apache.rediris.es/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
+RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
 ENV PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
 
 RUN cp "${SPARK_HOME}/conf/log4j.properties.template" "${SPARK_HOME}/conf/log4j.properties" && \
diff --git a/infra/docker/Makefile b/infra/docker/Makefile
index 75952f2..fcfd6a8 100644
--- a/infra/docker/Makefile
+++ b/infra/docker/Makefile
@@ -1,5 +1,5 @@
-SPARK_VERSION:=2.4.5
-COURSE_VERSION:=2020.1
+SPARK_VERSION:=3.0.1
+COURSE_VERSION:=2020.12
 IMAGE_NAME:=luisbelloch/spark
 
 .PHONY: help
diff --git a/local_setup.sh b/local_setup.sh
index 7c2a4f1..ebf1917 100755
--- a/local_setup.sh
+++ b/local_setup.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-SPARK_URL=${SPARK_URL:-http://apache.rediris.es/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz}
+SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz}
 SPARK_PKG=${SPARK_URL##*/}
 SPARK_HOME=${SPARK_HOME:-$(pwd)/.spark}
 
diff --git a/playbook.yml b/playbook.yml
index 58bdbbc..a463ce6 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -2,8 +2,8 @@
 - hosts: all
   vars:
     spark_home: /opt/spark
-    spark_pkg_name: spark-2.4.5-bin-hadoop2.7
-    spark_pkg_url: http://apache.rediris.es/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
+    spark_pkg_name: spark-3.0.1-bin-hadoop2.7
+    spark_pkg_url: https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
 
   tasks:
     - name: Update all packages to the latest version
@@ -20,7 +20,7 @@
         update_cache: yes
         force_apt_get: true
 
-    - name: Install AdoptOpenJDK 8
+    - name: Install AdoptOpenJDK 11
       become: true
       block:
         - name: Import keys
@@ -33,7 +33,7 @@
             state: present
         - name: Install package
           apt:
-            name: ['adoptopenjdk-8-hotspot', 'ca-certificates']
+            name: ['adoptopenjdk-11-hotspot', 'ca-certificates']
             state: latest
             update_cache: yes
             force_apt_get: true