Skip to content

Commit

Permalink
[SPARK-50421][CORE] Fix executor related memory config incorrect whe…
Browse files Browse the repository at this point in the history
…n multiple resource profiles worked
  • Loading branch information
zjuwangg committed Dec 6, 2024
1 parent 153cd9e commit d0311f0
Showing 1 changed file with 21 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,27 @@ private[spark] object CoarseGrainedExecutorBackend extends Logging {
}

driverConf.set(EXECUTOR_ID, arguments.executorId)
// Set executor memory related config here according to resource profile
if (cfg.resourceProfile.id != ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID) {
cfg.resourceProfile
.executorResources
.foreach {
case (ResourceProfile.OFFHEAP_MEM, request) =>
driverConf.set(MEMORY_OFFHEAP_SIZE.key, request.amount.toString + "m")
logInfo(s"Set executor off-heap memory to $request")
case (ResourceProfile.MEMORY, request) =>
driverConf.set(EXECUTOR_MEMORY.key, request.amount.toString + "m")
logInfo(s"Set executor memory to $request")
case (ResourceProfile.OVERHEAD_MEM, request) =>
// Maybe don't need to set this since it's nearly used by tasks.
driverConf.set(EXECUTOR_MEMORY_OVERHEAD.key, request.amount.toString + "m")
logInfo(s"Set executor memory_overhead to $request")
case (ResourceProfile.CORES, request) =>
driverConf.set(EXECUTOR_CORES.key, request.amount.toString)
logInfo(s"Set executor cores to $request")
case _ =>
}
}
val env = SparkEnv.createExecutorEnv(driverConf, arguments.executorId, arguments.bindAddress,
arguments.hostname, arguments.cores, cfg.ioEncryptionKey, isLocal = false)
// Set the application attemptId in the BlockStoreClient if available.
Expand Down

0 comments on commit d0311f0

Please sign in to comment.