From b72c02f06904af41a49ed20938217c0e449656f7 Mon Sep 17 00:00:00 2001 From: GuojingFeng Date: Tue, 11 Jul 2023 10:18:45 +0800 Subject: [PATCH] [SMALLFIX] Use field property for 'AbstractCmdRunner' ### What changes are proposed in this pull request? Use field property 'mActiveJobs' for 'AbstractCmdRunner' instead of the default value DEFAULT_ACTIVE_JOBS which is hardcoded with 3000. ### Why are the changes needed? Currently, the number of active jobs are limited at 3000 which is not enough when we have lots of loading tasks on huge cluster. We will add command line args to allow user pass the limit of active jobs (set the value of 'mActiveJobs') the in the future. ### Does this PR introduce any user facing changes? N/A pr-link: Alluxio/alluxio#17755 change-id: cid-cc87ac5731fac22724ca2d84ca3574f2a09033bb --- .../main/java/alluxio/master/job/tracker/DistLoadCliRunner.java | 2 +- .../main/java/alluxio/master/job/tracker/MigrateCliRunner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/job/server/src/main/java/alluxio/master/job/tracker/DistLoadCliRunner.java b/job/server/src/main/java/alluxio/master/job/tracker/DistLoadCliRunner.java index 40f3574d72cd..6d21cc5387fb 100644 --- a/job/server/src/main/java/alluxio/master/job/tracker/DistLoadCliRunner.java +++ b/job/server/src/main/java/alluxio/master/job/tracker/DistLoadCliRunner.java @@ -156,7 +156,7 @@ private void submitDistLoad(List pool, int replication, Set workerSet, Set excludedWorkerSet, Set localityIds, Set excludedLocalityIds, boolean directCache, CmdInfo cmdInfo) { - if (mSubmitted.size() >= DEFAULT_ACTIVE_JOBS) { + if (mSubmitted.size() >= mActiveJobs) { waitForCmdJob(); } diff --git a/job/server/src/main/java/alluxio/master/job/tracker/MigrateCliRunner.java b/job/server/src/main/java/alluxio/master/job/tracker/MigrateCliRunner.java index c72d4040ecec..c244409a9492 100644 --- a/job/server/src/main/java/alluxio/master/job/tracker/MigrateCliRunner.java +++ b/job/server/src/main/java/alluxio/master/job/tracker/MigrateCliRunner.java @@ -124,7 +124,7 @@ private void copy(AlluxioURI srcPath, AlluxioURI dstPath, boolean overwrite, int // Submit a child job within a distributed command job. private void submitDistCp(List> pool, boolean overwrite, WriteType writeType, CmdInfo cmdInfo) { - if (mSubmitted.size() >= DEFAULT_ACTIVE_JOBS) { + if (mSubmitted.size() >= mActiveJobs) { waitForCmdJob(); }