[SEDONA-651] Add spark prefix to all sedona spark config (#1580)

apache · Sep 5, 2024 · 5e64c84 · 5e64c84
1 parent bc396eb
commit 5e64c84
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 13 deletions.
diff --git a/docs/api/sql/Parameter.md b/docs/api/sql/Parameter.md
@@ -25,6 +25,16 @@ println(sedonaConf)
 sparkSession.conf.set("sedona.global.index","false")
 ```
 
+In addition, you can also add `spark` prefix to the parameter name, for example:
+
+```scala
+sparkSession.conf.set("spark.sedona.global.index","false")
+```
+
+However, any parameter set through `spark` prefix will be honored by Spark, which means you can set these parameters before hand via `spark-defaults.conf` or Spark on Kubernetes configuration.
+
+If you set the same parameter through both `sedona` and `spark.sedona` prefixes, the parameter set through `sedona` prefix will override the parameter set through `spark.sedona` prefix.
+
 ## Explanation
 
 * sedona.global.index

diff --git a/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java b/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java
@@ -64,38 +64,53 @@ public static SedonaConf fromActiveSession() {
   }
 
   public SedonaConf(RuntimeConfig runtimeConfig) {
-    this.useIndex = Boolean.parseBoolean(runtimeConfig.get("sedona.global.index", "true"));
-    this.indexType = IndexType.getIndexType(runtimeConfig.get("sedona.global.indextype", "rtree"));
+    this.useIndex = Boolean.parseBoolean(getConfigValue(runtimeConfig, "global.index", "true"));
+    this.indexType =
+        IndexType.getIndexType(getConfigValue(runtimeConfig, "global.indextype", "rtree"));
     this.joinApproximateTotalCount =
-        Long.parseLong(runtimeConfig.get("sedona.join.approxcount", "-1"));
-    String[] boundaryString = runtimeConfig.get("sedona.join.boundary", "0,0,0,0").split(",");
+        Long.parseLong(getConfigValue(runtimeConfig, "join.approxcount", "-1"));
+    String[] boundaryString = getConfigValue(runtimeConfig, "join.boundary", "0,0,0,0").split(",");
     this.datasetBoundary =
         new Envelope(
             Double.parseDouble(boundaryString[0]),
             Double.parseDouble(boundaryString[1]),
             Double.parseDouble(boundaryString[2]),
             Double.parseDouble(boundaryString[3]));
-    this.joinGridType = GridType.getGridType(runtimeConfig.get("sedona.join.gridtype", "kdbtree"));
+    this.joinGridType =
+        GridType.getGridType(getConfigValue(runtimeConfig, "join.gridtype", "kdbtree"));
     this.joinBuildSide =
-        JoinBuildSide.getBuildSide(runtimeConfig.get("sedona.join.indexbuildside", "left"));
+        JoinBuildSide.getBuildSide(getConfigValue(runtimeConfig, "join.indexbuildside", "left"));
     this.joinSparitionDominantSide =
         JoinSparitionDominantSide.getJoinSparitionDominantSide(
-            runtimeConfig.get("sedona.join.spatitionside", "left"));
+            getConfigValue(runtimeConfig, "join.spatitionside", "left"));
     this.fallbackPartitionNum =
-        Integer.parseInt(runtimeConfig.get("sedona.join.numpartition", "-1"));
+        Integer.parseInt(getConfigValue(runtimeConfig, "join.numpartition", "-1"));
     this.autoBroadcastJoinThreshold =
         bytesFromString(
-            runtimeConfig.get(
-                "sedona.join.autoBroadcastJoinThreshold",
+            getConfigValue(
+                runtimeConfig,
+                "join.autoBroadcastJoinThreshold",
                 runtimeConfig.get("spark.sql.autoBroadcastJoinThreshold")));
     this.spatialJoinOptimizationMode =
         SpatialJoinOptimizationMode.getSpatialJoinOptimizationMode(
-            runtimeConfig.get("sedona.join.optimizationmode", "nonequi"));
+            getConfigValue(runtimeConfig, "join.optimizationmode", "nonequi"));
 
     // Parameters for knn joins
     this.includeTieBreakersInKNNJoins =
-        Boolean.parseBoolean(
-            runtimeConfig.get("spark.sedona.join.knn.includeTieBreakers", "false"));
+        Boolean.parseBoolean(getConfigValue(runtimeConfig, "join.knn.includeTieBreakers", "false"));
+  }
+
+  // Helper method to prioritize `sedona.*` over `spark.sedona.*`
+  private String getConfigValue(
+      RuntimeConfig runtimeConfig, String keySuffix, String defaultValue) {
+    String sedonaKey = "sedona." + keySuffix;
+    String sparkSedonaKey = "spark.sedona." + keySuffix;
+
+    if (runtimeConfig.contains(sedonaKey)) {
+      return runtimeConfig.get(sedonaKey, defaultValue);
+    } else {
+      return runtimeConfig.get(sparkSedonaKey, defaultValue);
+    }
   }
 
   public boolean getUseIndex() {