Refactor arguments to handle options after event logs (#466)

Signed-off-by: Partho Sarthi <[email protected]>
NVIDIA · Jul 28, 2023 · 8e43055 · 8e43055
1 parent e5bfaa7
commit 8e43055
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 2 deletions.
diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileArgs.scala
@@ -18,9 +18,10 @@ package com.nvidia.spark.rapids.tool.profiling
 import org.rogach.scallop.{ScallopConf, ScallopOption}
 import org.rogach.scallop.exceptions.ScallopException
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.rapids.tool.AppFilterImpl
 
-class ProfileArgs(arguments: Seq[String]) extends ScallopConf(arguments) {
+class ProfileArgs(arguments: Seq[String]) extends ScallopConf(arguments) with Logging {
 
   banner("""
 Profiling Tool for the RAPIDS Accelerator and Apache Spark
@@ -122,6 +123,26 @@ Usage: java -cp rapids-4-spark-tools_2.12-<version>.jar:$SPARK_HOME/jars/*
       "are min(minute),h(hours),d(days),w(weeks),m(months).")
   }
 
+  /**
+  * Assumption: Event-log files do not begin with a '-'.
+  *
+  * Inform the user that all options after the event-logs will be ignored.
+  * Eg,  Arguments: '--output-directory result_dir /path-to-log --timeout 50 --num-threads 100'
+  * Result: Option '--timeout 50 --num-threads 100' will be ignored.
+  */
+  validate(eventlog) { log =>
+    // `log/eventlog` is a trailing argument.
+    // Drop all elements in it until the first occurrence of '-'.
+    val ignoredOptions = log.dropWhile(s => !s.startsWith("-"))
+    // If any elements exist in the `ignored` list, these are additional options that
+    // will be skipped by EventLogPathProcessor
+    if (ignoredOptions.nonEmpty) {
+      logWarning(s"Options provided after event logs will be ignored: " +
+        s"${ignoredOptions.mkString(" ")}")
+    }
+    Right(Unit)
+  }
+
   verify()
 
   override def onError(e: Throwable) = e match {

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualificationArgs.scala
@@ -18,9 +18,10 @@ package com.nvidia.spark.rapids.tool.qualification
 import org.rogach.scallop.{ScallopConf, ScallopOption}
 import org.rogach.scallop.exceptions.ScallopException
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.rapids.tool.AppFilterImpl
 
-class QualificationArgs(arguments: Seq[String]) extends ScallopConf(arguments) {
+class QualificationArgs(arguments: Seq[String]) extends ScallopConf(arguments) with Logging {
 
   banner("""
 RAPIDS Accelerator for Apache Spark qualification tool
@@ -176,6 +177,26 @@ Usage: java -cp rapids-4-spark-tools_2.12-<version>.jar:$SPARK_HOME/jars/*
       "are min(minute),h(hours),d(days),w(weeks),m(months).")
   }
 
+  /**
+   * Assumption: Event-log files do not begin with a '-'.
+   *
+   * Inform the user that all options after the event-logs will be ignored.
+   * Eg,  Arguments: '--output-directory result_dir /path-to-log --timeout 50 --num-threads 100'
+   * Result: Option '--timeout 50 --num-threads 100' will be ignored.
+   */
+  validate(eventlog) { log =>
+    // `log/eventlog` is a trailing argument.
+    // Drop all elements in it until the first occurrence of '-'.
+    val ignoredOptions = log.dropWhile(s => !s.startsWith("-"))
+    // If any elements exist in the `ignored` list, these are additional options that
+    // will be skipped by EventLogPathProcessor
+    if (ignoredOptions.nonEmpty) {
+      logWarning(s"Options provided after event logs will be ignored: " +
+        s"${ignoredOptions.mkString(" ")}")
+    }
+    Right(Unit)
+  }
+
   verify()
 
   override def onError(e: Throwable) = e match {