From 84f8e9b75a511eb394e9559428d1872997da58e5 Mon Sep 17 00:00:00 2001 From: Julien Richard-Foy Date: Fri, 23 Aug 2024 09:20:20 +0200 Subject: [PATCH] Use --driver-memory instead of --conf spark.driver.memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to Spark documentation, the configuration property `spark.driver.memory` has no effect in our case (we use the “client” deploy-mode): > In client mode, this config must not be set through the SparkConf directly in your application, because the driver JVM has already started at that point. Instead, please set this through the --driver-memory command line option or in your default properties file. https://spark.apache.org/docs/latest/configuration.html --- ansible/templates/submit-alternator-job.sh | 2 +- ansible/templates/submit-cql-job.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/templates/submit-alternator-job.sh b/ansible/templates/submit-alternator-job.sh index 7acbdb0..f9d554b 100644 --- a/ansible/templates/submit-alternator-job.sh +++ b/ansible/templates/submit-alternator-job.sh @@ -12,5 +12,5 @@ time spark-submit --class com.scylladb.migrator.Migrator \ --conf spark.scylla.config=/home/ubuntu/scylla-migrator/config.dynamodb.yml \ --executor-memory $EXECUTOR_MEMORY \ --executor-cores $EXECUTOR_CORES \ ---conf spark.driver.memory=64G \ +--driver-memory 4G \ /home/ubuntu/scylla-migrator/scylla-migrator-assembly.jar \ No newline at end of file diff --git a/ansible/templates/submit-cql-job.sh b/ansible/templates/submit-cql-job.sh index e9f6143..f167df7 100644 --- a/ansible/templates/submit-cql-job.sh +++ b/ansible/templates/submit-cql-job.sh @@ -19,7 +19,7 @@ time spark-submit --class com.scylladb.migrator.Migrator \ #sometimes you will need a tuning for driver memory size #add this config to above to tune it: -# --conf spark.driver.memory=4G \ +# --driver-memory 4G \ # debug example #$SPARK_HOME/spark-submit --class com.scylladb.migrator.Migrator \