diff --git a/h2o-hadoop-2/assemblyjar.gradle b/h2o-hadoop-2/assemblyjar.gradle index 973231889b58..6b69dce78b85 100644 --- a/h2o-hadoop-2/assemblyjar.gradle +++ b/h2o-hadoop-2/assemblyjar.gradle @@ -7,7 +7,7 @@ sourceCompatibility = 1.8 targetCompatibility = 1.8 ext { - parquetVersion = "1.8.1" + defaultParquetVersion = "1.8.1" } def hasCustomHdfsDep = ext.has("hdfsDependency") && (ext.get("hdfsDependency") != defaultHdfsDependency) @@ -56,10 +56,7 @@ dependencies { api(project(":h2o-ext-krbstandalone")) { transitive = false // we just need hadoop-auth which is provided by the hadoop-client dependency } - api("org.apache.parquet:parquet-avro:${parquetVersion}") // required by h2o-hive - api("org.apache.parquet:parquet-hadoop:${parquetVersion}") { // required by h2o-parquet-parser - force = true - } + api("org.apache.parquet:parquet-avro:${defaultParquetVersion}") // required by h2o-hive // For standalone mode to work with MapR, this extra library needs to be // included, and it's not pulled in by the dependency stuff; this must @@ -78,9 +75,13 @@ dependencies { transitive = false } } - api(project(":h2o-parquet-parser")) + api(project(":h2o-parquet-parser")) { + transitive = false + } } +apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle" + // // Bundle optional modules // The process is defined by convention. There are two flags: diff --git a/h2o-hadoop-3/assemblyjar_common.gradle b/h2o-hadoop-3/assemblyjar_common.gradle index d02252c5ff53..ce5b3fc8c039 100644 --- a/h2o-hadoop-3/assemblyjar_common.gradle +++ b/h2o-hadoop-3/assemblyjar_common.gradle @@ -1,5 +1,5 @@ ext { - parquetVersion = "1.8.1" + defaultParquetVersion = "1.8.1" } dependencies { @@ -19,11 +19,7 @@ dependencies { api(project(":h2o-ext-krbstandalone")) { transitive = false // we just need hadoop-auth which is provided by the hadoop-client dependency } - api("org.apache.parquet:parquet-avro:${parquetVersion}") // required by h2o-hive - api("org.apache.parquet:parquet-hadoop:${parquetVersion}") { // required by h2o-parquet-parser - force = true - } - + api("org.apache.parquet:parquet-avro:${defaultParquetVersion}") // required by h2o-hive // For standalone mode to work with MapR, this extra library needs to be // included, and it's not pulled in by the dependency stuff; this must // be a bug in MapR's packaging process. @@ -41,9 +37,14 @@ dependencies { transitive = false } } - api(project(":h2o-parquet-parser")) + api(project(":h2o-parquet-parser")) { + transitive = false + } + } +apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle" + // // Bundle optional modules // The process is defined by convention. There are two flags: diff --git a/h2o-parsers/h2o-parquet-parser/build.gradle b/h2o-parsers/h2o-parquet-parser/build.gradle index a7fdb8829c84..79d171bd5b5a 100644 --- a/h2o-parsers/h2o-parquet-parser/build.gradle +++ b/h2o-parsers/h2o-parquet-parser/build.gradle @@ -3,46 +3,13 @@ // description = "H2O Parquet Parser" -def parquetHadoopVersion = binding.variables.get("hadoopVersion") ? - binding.variables.get("hadoopVersion") : defaultHadoopVersion - -configurations{ - // Configuration used to get all transitive dependencies for org.apache.hadoop:hadoop-common - hadoopCommonExclude -} - dependencies { - hadoopCommonExclude("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") - - api project(":h2o-core") - api(project(":h2o-persist-hdfs")) { - exclude group: 'ai.h2o', module: 'h2o-core' - exclude group: 'net.java.dev.jets3t', module: 'jets3t' - exclude group: 'org.apache.hadoop', module: 'hadoop-client' - exclude group: 'org.apache.hadoop', module: 'hadoop-aws' - } - - // Parquet support - api("org.apache.parquet:parquet-hadoop:${defaultParquetVersion}") - - - api("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") { - // we can't use transitive=false so we need to exclude the dependencies manually - configurations.hadoopCommonExclude.getResolvedConfiguration().getResolvedArtifacts().each { - if (it.moduleVersion.id.group != "org.apache.hadoop" && it.moduleVersion.id.module.name != "hadoop-common") { - exclude group: it.moduleVersion.id.group, module: it.moduleVersion.id.module.name - } - } - } - implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${parquetHadoopVersion}") { - transitive = false - } - testImplementation project(":h2o-test-support") testImplementation project(":h2o-parquet-parser-tests") testRuntimeOnly project(":${defaultWebserverModule}") } +apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle" apply from: "${rootDir}/gradle/dataCheck.gradle" test { diff --git a/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle b/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle new file mode 100644 index 000000000000..4abb69601464 --- /dev/null +++ b/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle @@ -0,0 +1,35 @@ +def parquetHadoopVersion = binding.variables.get("hadoopVersion") ? + binding.variables.get("hadoopVersion") : defaultHadoopVersion + +configurations{ + // Configuration used to get all transitive dependencies for org.apache.hadoop:hadoop-common + hadoopCommonExclude +} + +dependencies { + hadoopCommonExclude("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") + + api project(":h2o-core") + api(project(":h2o-persist-hdfs")) { + exclude group: 'ai.h2o', module: 'h2o-core' + exclude group: 'net.java.dev.jets3t', module: 'jets3t' + exclude group: 'org.apache.hadoop', module: 'hadoop-client' + exclude group: 'org.apache.hadoop', module: 'hadoop-aws' + } + + // Parquet support + api("org.apache.parquet:parquet-hadoop:${defaultParquetVersion}") + + + api("org.apache.hadoop:hadoop-common:${parquetHadoopVersion}") { + // we can't use transitive=false so we need to exclude the dependencies manually + configurations.hadoopCommonExclude.getResolvedConfiguration().getResolvedArtifacts().each { + if (it.moduleVersion.id.group != "org.apache.hadoop" && it.moduleVersion.id.module.name != "hadoop-common") { + exclude group: it.moduleVersion.id.group, module: it.moduleVersion.id.module.name + } + } + } + implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${parquetHadoopVersion}") { + transitive = false + } +}