diff --git a/main.tf b/main.tf index e9eac35..73f7d86 100644 --- a/main.tf +++ b/main.tf @@ -36,6 +36,8 @@ locals { spark_dynamic_allocation_enabled = var.spark_dynamic_allocation_enabled spark_max_executors = var.spark_max_executors spark_min_executors = var.spark_min_executors + spark_version = var.zeppelin_version + spark_mirror = var.zeppelin_mirror } ) }, diff --git a/templates/cloud_config.yaml b/templates/cloud_config.yaml index fd0286f..cf70865 100644 --- a/templates/cloud_config.yaml +++ b/templates/cloud_config.yaml @@ -242,10 +242,10 @@ runcmd: - resolvconf -u #Install Spark - cd /opt - - wget https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz - - tar xzf spark-3.5.1-bin-hadoop3.tgz - - mv spark-3.5.1-bin-hadoop3 spark - - rm spark-3.5.1-bin-hadoop3.tgz + - wget ${spark_mirror}/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop3.tgz + - tar xzf spark-${spark_version}-bin-hadoop3.tgz + - mv spark-${spark_version}-bin-hadoop3 spark + - rm spark-${spark_version}-bin-hadoop3.tgz - cp /opt/spark-defaults.conf /opt/spark/conf/spark-defaults.conf #Install Zeppelin - cd /opt diff --git a/variables.tf b/variables.tf index c671823..b22fd56 100644 --- a/variables.tf +++ b/variables.tf @@ -168,6 +168,18 @@ variable "hive_metastore_url" { type = string } +variable "spark_version" { + description = "Version of spark" + type = string + default = "3.5.2" +} + +variable "spark_mirror" { + description = "Mirror from which to download spark" + type = string + default = "https://dlcdn.apache.org" +} + variable "spark_sql_warehouse_dir" { description = "S3 path of the spark sql warehouse" type = string