feat: INFRA-715 Upgrade spark

main.tf

@@ -36,6 +36,8 @@ locals {
           spark_dynamic_allocation_enabled = var.spark_dynamic_allocation_enabled
           spark_max_executors              = var.spark_max_executors
           spark_min_executors              = var.spark_min_executors
+          spark_version                 = var.zeppelin_version
+          spark_mirror                  = var.zeppelin_mirror
         }
       )
     },

templates/cloud_config.yaml

@@ -242,10 +242,10 @@ runcmd:
   - resolvconf -u
   #Install Spark
   - cd /opt
-  - wget https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz
-  - tar xzf spark-3.5.1-bin-hadoop3.tgz
-  - mv spark-3.5.1-bin-hadoop3 spark
-  - rm spark-3.5.1-bin-hadoop3.tgz
+  - wget ${spark_mirror}/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop3.tgz
+  - tar xzf spark-${spark_version}-bin-hadoop3.tgz
+  - mv spark-${spark_version}-bin-hadoop3 spark
+  - rm spark-${spark_version}-bin-hadoop3.tgz
   - cp /opt/spark-defaults.conf /opt/spark/conf/spark-defaults.conf
   #Install Zeppelin
   - cd /opt

variables.tf

@@ -168,6 +168,18 @@ variable "hive_metastore_url" {
   type        = string
 }
 
+variable "spark_version" {
+  description = "Version of spark"
+  type        = string
+  default     = "3.5.2"
+}
+
+variable "spark_mirror" {
+  description = "Mirror from which to download spark"
+  type        = string
+  default     = "https://dlcdn.apache.org"
+}
+
 variable "spark_sql_warehouse_dir" {
   description = "S3 path of the spark sql warehouse"
   type        = string