Skip to content

Commit

Permalink
[PUBDEV-8985] Fix CDH 6.1 kerberos tests (#15591)
Browse files Browse the repository at this point in the history
  • Loading branch information
mn-mikke committed Jun 19, 2023
1 parent 1619acd commit 7bef068
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 47 deletions.
13 changes: 7 additions & 6 deletions h2o-hadoop-2/assemblyjar.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ sourceCompatibility = 1.8
targetCompatibility = 1.8

ext {
parquetVersion = "1.8.1"
defaultParquetVersion = "1.8.1"
}

def hasCustomHdfsDep = ext.has("hdfsDependency") && (ext.get("hdfsDependency") != defaultHdfsDependency)
Expand Down Expand Up @@ -56,10 +56,7 @@ dependencies {
api(project(":h2o-ext-krbstandalone")) {
transitive = false // we just need hadoop-auth which is provided by the hadoop-client dependency
}
api("org.apache.parquet:parquet-avro:${parquetVersion}") // required by h2o-hive
api("org.apache.parquet:parquet-hadoop:${parquetVersion}") { // required by h2o-parquet-parser
force = true
}
api("org.apache.parquet:parquet-avro:${defaultParquetVersion}") // required by h2o-hive

// For standalone mode to work with MapR, this extra library needs to be
// included, and it's not pulled in by the dependency stuff; this must
Expand All @@ -78,9 +75,13 @@ dependencies {
transitive = false
}
}
api(project(":h2o-parquet-parser"))
api(project(":h2o-parquet-parser")) {
transitive = false
}
}

apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle"

//
// Bundle optional modules
// The process is defined by convention. There are two flags:
Expand Down
15 changes: 8 additions & 7 deletions h2o-hadoop-3/assemblyjar_common.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ext {
parquetVersion = "1.8.1"
defaultParquetVersion = "1.8.1"
}

dependencies {
Expand All @@ -19,11 +19,7 @@ dependencies {
api(project(":h2o-ext-krbstandalone")) {
transitive = false // we just need hadoop-auth which is provided by the hadoop-client dependency
}
api("org.apache.parquet:parquet-avro:${parquetVersion}") // required by h2o-hive
api("org.apache.parquet:parquet-hadoop:${parquetVersion}") { // required by h2o-parquet-parser
force = true
}

api("org.apache.parquet:parquet-avro:${defaultParquetVersion}") // required by h2o-hive
// For standalone mode to work with MapR, this extra library needs to be
// included, and it's not pulled in by the dependency stuff; this must
// be a bug in MapR's packaging process.
Expand All @@ -41,9 +37,14 @@ dependencies {
transitive = false
}
}
api(project(":h2o-parquet-parser"))
api(project(":h2o-parquet-parser")) {
transitive = false
}

}

apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle"

//
// Bundle optional modules
// The process is defined by convention. There are two flags:
Expand Down
35 changes: 1 addition & 34 deletions h2o-parsers/h2o-parquet-parser/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,13 @@
//
description = "H2O Parquet Parser"

def parquetHadoopVersion = binding.variables.get("hadoopVersion") ?
binding.variables.get("hadoopVersion") : defaultHadoopVersion

configurations{
// Configuration used to get all transitive dependencies for org.apache.hadoop:hadoop-common
hadoopCommonExclude
}

dependencies {
hadoopCommonExclude("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}")

api project(":h2o-core")
api(project(":h2o-persist-hdfs")) {
exclude group: 'ai.h2o', module: 'h2o-core'
exclude group: 'net.java.dev.jets3t', module: 'jets3t'
exclude group: 'org.apache.hadoop', module: 'hadoop-client'
exclude group: 'org.apache.hadoop', module: 'hadoop-aws'
}

// Parquet support
api("org.apache.parquet:parquet-hadoop:${defaultParquetVersion}")


api("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") {
// we can't use transitive=false so we need to exclude the dependencies manually
configurations.hadoopCommonExclude.getResolvedConfiguration().getResolvedArtifacts().each {
if (it.moduleVersion.id.group != "org.apache.hadoop" && it.moduleVersion.id.module.name != "hadoop-common") {
exclude group: it.moduleVersion.id.group, module: it.moduleVersion.id.module.name
}
}
}
implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${parquetHadoopVersion}") {
transitive = false
}

testImplementation project(":h2o-test-support")
testImplementation project(":h2o-parquet-parser-tests")
testRuntimeOnly project(":${defaultWebserverModule}")
}

apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle"
apply from: "${rootDir}/gradle/dataCheck.gradle"

test {
Expand Down
35 changes: 35 additions & 0 deletions h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
def parquetHadoopVersion = binding.variables.get("hadoopVersion") ?
binding.variables.get("hadoopVersion") : defaultHadoopVersion

configurations{
// Configuration used to get all transitive dependencies for org.apache.hadoop:hadoop-common
hadoopCommonExclude
}

dependencies {
hadoopCommonExclude("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}")

api project(":h2o-core")
api(project(":h2o-persist-hdfs")) {
exclude group: 'ai.h2o', module: 'h2o-core'
exclude group: 'net.java.dev.jets3t', module: 'jets3t'
exclude group: 'org.apache.hadoop', module: 'hadoop-client'
exclude group: 'org.apache.hadoop', module: 'hadoop-aws'
}

// Parquet support
api("org.apache.parquet:parquet-hadoop:${defaultParquetVersion}")


api("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") {
// we can't use transitive=false so we need to exclude the dependencies manually
configurations.hadoopCommonExclude.getResolvedConfiguration().getResolvedArtifacts().each {
if (it.moduleVersion.id.group != "org.apache.hadoop" && it.moduleVersion.id.module.name != "hadoop-common") {
exclude group: it.moduleVersion.id.group, module: it.moduleVersion.id.module.name
}
}
}
implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${parquetHadoopVersion}") {
transitive = false
}
}

0 comments on commit 7bef068

Please sign in to comment.