spark-env.sh.erb

#!/usr/bin/env bash

# This file contains environment variables required to run Spark. Copy it as
# spark-env.sh and edit that to configure Spark for your site. At a minimum,
# the following two variables should be set:
# - MESOS_NATIVE_LIBRARY, to point to your Mesos native library (libmesos.so)
# - SCALA_HOME, to point to your Scala installation
#
# If using the standalone deploy mode, you can also set variables for it:
# - SPARK_MASTER_IP, to bind the master to a different IP address
# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports
# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
# - SPARK_WORKER_MEMORY, to set how much memory to use (e.g. 1000m, 2g)
# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT
# - SPARK_WORKER_INSTANCES, to set the number of worker instances/processes to be spawned on every slave machine
#
# Finally, Spark also relies on the following variables, but these can be set
# on just the *master* (i.e. in your driver program), and will automatically
# be propagated to workers:
# - SPARK_MEM, to change the amount of memory used per node (this should
#   be in the same format as the JVM's -Xmx option, e.g. 300m or 1g)
# - SPARK_CLASSPATH, to add elements to Spark's classpath
# - SPARK_JAVA_OPTS, to add JVM options
# - SPARK_LIBRARY_PATH, to add extra search paths for native libraries.

export MESOS_NATIVE_LIBRARY="<%= node[:datastax][:dse][:spark][:mesos_native_library] %>"
export SCALA_HOME="<%= node[:datastax][:dse][:spark][:mesos_native_library] %>"

export SPARK_MASTER_PORT=<%= node[:datastax][:dse][:spark][:master_port] %>
export SPARK_MASTER_WEBUI_PORT=<%= node[:datastax][:dse][:spark][:master_webui_port] %>
export SPARK_WORKER_WEBUI_PORT=<%= node[:datastax][:dse][:spark][:worker_webui_port] %>

# By default, Spark Worker uses nearly all available resources in the system
# export SPARK_WORKER_CORES="2"
export SPARK_WORKER_MEMORY="<%= node[:datastax][:dse][:spark][:worker_memory] %>"
export SPARK_MEM="<%= node[:datastax][:dse][:spark][:mem] %>"
export SPARK_REPL_MEM="<%= node[:datastax][:dse][:spark][:repl_mem] %>"

export SPARK_TMP_DIR="<%= node[:datastax][:dse][:spark][:tmp_dir] %>"
export SPARK_LOG_DIR="<%= node[:datastax][:dse][:spark][:logs_dir] %>"

export SPARK_COMMON_OPTS="$SPARK_COMMON_OPTS <%= node[:datastax][:dse][:spark][:common_opts] %> "

# Individual Java opts for different Spark components
export SPARK_MASTER_OPTS=" <%= node[:datastax][:dse][:spark][:master_opts] %> "
export SPARK_WORKER_OPTS=" <%= node[:datastax][:dse][:spark][:worker_opts] %> "
export SPARK_EXECUTOR_OPTS=" <%= node[:datastax][:dse][:spark][:executor_opts] %> "
export SPARK_REPL_OPTS=" <%= node[:datastax][:dse][:spark][:repl_opts] %> "
export SPARK_APP_OPTS=" <%= node[:datastax][:dse][:spark][:app_opts] %> "

# Directory to run applications in, which will include both logs and scratch space (default: SPARK_HOME/work).
export SPARK_WORKER_DIR="<%= node[:datastax][:dse][:spark][:worker_dir] %>"