From 3b9f25b62d43600274af89fccfec0ecc31283f1c Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 22 Mar 2024 10:30:30 +0800 Subject: [PATCH] [KYUUBI #6197] Revise dependency management of Spark authZ plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description ## Issue References ๐Ÿ”— The POM of `kyuubi-spark-authz-shaded` is redundant, just pull `kyuubi-spark-authz` is necessary. The current dependency management does not work on Ranger 2.1.0, this patch cleans up the POM definition and fixes the compatibility with Ranger 2.1.0 ## Describe Your Solution ๐Ÿ”ง Carefully revise the dependency list and exclusion. ## Types of changes :bookmark: - [x] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช perform packing kyuubi-spark-authz-shaded module. ``` build/mvn clean install -pl extensions/spark/kyuubi-spark-authz-shaded -am -DskipTests ``` before ``` [INFO] --- maven-shade-plugin:3.5.2:shade (default) kyuubi-spark-authz-shaded_2.12 --- [INFO] Including org.apache.kyuubi:kyuubi-spark-authz_2.12:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.kyuubi:kyuubi-util-scala_2.12:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.kyuubi:kyuubi-util:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-common:jar:2.4.0 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-jaxrs:jar:1.9.13 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-core-asl:jar:1.9.13 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13 in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-cred:jar:2.4.0 in the shaded jar. [INFO] Including com.sun.jersey:jersey-client:jar:1.19.4 in the shaded jar. [INFO] Including com.sun.jersey:jersey-core:jar:1.19.4 in the shaded jar. [INFO] Including com.kstruct:gethostname4j:jar:1.0.0 in the shaded jar. [INFO] Including net.java.dev.jna:jna:jar:5.7.0 in the shaded jar. [INFO] Including net.java.dev.jna:jna-platform:jar:5.7.0 in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-audit:jar:2.4.0 in the shaded jar. ``` after ``` [INFO] --- maven-shade-plugin:3.5.2:shade (default) kyuubi-spark-authz-shaded_2.12 --- [INFO] Including org.apache.kyuubi:kyuubi-spark-authz_2.12:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.kyuubi:kyuubi-util-scala_2.12:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.kyuubi:kyuubi-util:jar:1.10.0-SNAPSHOT in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-common:jar:2.4.0 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-jaxrs:jar:1.9.13 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-core-asl:jar:1.9.13 in the shaded jar. [INFO] Including org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13 in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-cred:jar:2.4.0 in the shaded jar. [INFO] Including com.sun.jersey:jersey-client:jar:1.19.4 in the shaded jar. [INFO] Including com.sun.jersey:jersey-core:jar:1.19.4 in the shaded jar. [INFO] Including com.kstruct:gethostname4j:jar:1.0.0 in the shaded jar. [INFO] Including net.java.dev.jna:jna:jar:5.7.0 in the shaded jar. [INFO] Including net.java.dev.jna:jna-platform:jar:5.7.0 in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugin-classloader:jar:2.4.0 in the shaded jar. [INFO] Including org.apache.ranger:ranger-plugins-audit:jar:2.4.0 in the shaded jar. ``` --- # Checklist ๐Ÿ“ - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6197 from pan3793/authz-dep. Closes #6197 d0becabce [Cheng Pan] 2.4 47e38502a [Cheng Pan] ranger 2.4 af01f7ed5 [Cheng Pan] test ranger 2.1 203aff3b3 [Cheng Pan] ranger-plugins-cred 974d76b03 [Cheng Pan] Resive dependency management of authz e5154f30f [Cheng Pan] improve authz deps Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .../spark/kyuubi-spark-authz-shaded/pom.xml | 210 +----------------- extensions/spark/kyuubi-spark-authz/pom.xml | 85 ++++--- pom.xml | 1 - 3 files changed, 60 insertions(+), 236 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz-shaded/pom.xml b/extensions/spark/kyuubi-spark-authz-shaded/pom.xml index 4ff50edbc32..d7bfd3db819 100644 --- a/extensions/spark/kyuubi-spark-authz-shaded/pom.xml +++ b/extensions/spark/kyuubi-spark-authz-shaded/pom.xml @@ -30,197 +30,12 @@ Kyuubi Dev Spark Authorization Extension Shaded https://kyuubi.apache.org/ - - - 1.0.0 - 1.19.4 - 5.7.0 - - org.apache.kyuubi kyuubi-spark-authz_${scala.binary.version} ${project.version} - - org.apache.kyuubi - kyuubi-util-scala_${scala.binary.version} - ${project.version} - - - org.apache.ranger - ranger-plugins-common - ${ranger.version} - - - com.sun.jersey - jersey-bundle - - - org.apache.ranger - ranger-plugin-classloader - - - org.apache.ranger - ranger-plugins-audit - - - log4j - log4j - - - ch.qos.logback - logback-classic - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - org.apache.httpcomponents - * - - - org.apache.hadoop - hadoop-common - - - javax.ws.rs - jsr311-api - - - com.kstruct - gethostname4j - - - net.java.dev.jna - jna - - - net.java.dev.jna - jna-platform - - - - - - com.sun.jersey - jersey-client - ${jersey.client.version} - - - javax.ws.rs - jsr311-api - - - - - - com.kstruct - gethostname4j - ${gethostname4j.version} - - - - net.java.dev.jna - jna - ${jna.version} - - - - net.java.dev.jna - jna-platform - ${jna.version} - - - - org.apache.ranger - ranger-plugins-audit - ${ranger.version} - - - org.apache.ranger - ranger-plugins-cred - - - org.apache.kafka - * - - - org.apache.solr - solr-solrj - - - org.elasticsearch - * - - - org.elasticsearch.client - * - - - org.elasticsearch.plugin - * - - - org.apache.lucene - * - - - log4j - log4j - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - com.carrotsearch - hppc - - - org.apache.httpcomponents - * - - - org.apache.hive - hive-storage-api - - - org.apache.orc - orc-core - - - org.apache.hadoop - hadoop-common - - - com.google.guava - guava - - - joda-time - joda-time - - - org.apache.logging.log4j - * - - - com.amazonaws - aws-java-sdk-bundle - - - @@ -232,18 +47,12 @@ false - org.apache.kyuubi:kyuubi-util-scala_${scala.binary.version} - org.apache.kyuubi:kyuubi-spark-authz_${scala.binary.version} - org.apache.kyuubi:kyuubi-util - org.apache.ranger:ranger-plugins-common - org.apache.ranger:ranger-plugins-audit - org.apache.ranger:ranger-plugins-cred - org.codehaus.jackson:jackson-jaxrs - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.sun.jersey:jersey-client - com.sun.jersey:jersey-core + org.apache.kyuubi:* + org.apache.ranger:* + org.codehaus.jackson:* + com.sun.jersey:* com.kstruct:gethostname4j + net.java.dev.jna:jna net.java.dev.jna:jna-platform @@ -287,15 +96,6 @@ com.kstruct.gethostname4j ${kyuubi.shade.packageName}.com.kstruct.gethostname4j - - org.apache.hadoop.security - ${kyuubi.shade.packageName}.org.apache.hadoop.security - - org.apache.hadoop.security.KrbPasswordSaverLoginModule - org.apache.hadoop.security.SecureClientLogin - org.apache.hadoop.security.SecureClientLoginConfiguration - - diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml index 7c6e30d1da1..978dad1431b 100644 --- a/extensions/spark/kyuubi-spark-authz/pom.xml +++ b/extensions/spark/kyuubi-spark-authz/pom.xml @@ -32,6 +32,7 @@ https://kyuubi.apache.org/ + 2.4.0 1.0.0 1.19.4 @@ -49,10 +50,6 @@ ranger-plugins-common ${ranger.version} - - com.sun.jersey - jersey-bundle - org.apache.ranger ranger-plugin-classloader @@ -61,6 +58,11 @@ org.apache.ranger ranger-plugins-audit + + + com.sun.jersey + jersey-bundle + log4j log4j @@ -70,17 +72,13 @@ logback-classic - commons-lang - commons-lang + org.apache.commons + commons-configuration2 commons-logging commons-logging - - org.apache.httpcomponents - * - org.apache.hadoop hadoop-common @@ -89,14 +87,6 @@ javax.ws.rs jsr311-api - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - com.kstruct gethostname4j @@ -142,6 +132,12 @@ ${jna.version} + + org.apache.ranger + ranger-plugin-classloader + ${ranger.version} + + org.apache.ranger ranger-plugins-audit @@ -226,6 +222,35 @@ + + org.apache.ranger + ranger-plugins-cred + ${ranger.version} + + + org.apache.commons + commons-configuration2 + + + org.apache.hadoop + hadoop-common + + + log4j + log4j + + + + org.eclipse.persistence + javax.persistence + + + org.eclipse.persistence + eclipselink + + + + org.scala-lang scala-library @@ -256,18 +281,6 @@ provided - - org.apache.logging.log4j - log4j-api - provided - - - - org.apache.logging.log4j - log4j-core - provided - - commons-collections commons-collections @@ -294,6 +307,18 @@ + + org.apache.logging.log4j + log4j-api + test + + + + org.apache.logging.log4j + log4j-core + test + + org.apache.logging.log4j log4j-slf4j-impl diff --git a/pom.xml b/pom.xml index 3f607562656..8aeebd4df02 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,6 @@ 0.16.0 3.21.7 0.10.7 - 2.4.0 3.2.16 3.2.16.0 4.1.0