Skip to content

Commit

Permalink
Disable unsupported MapR profile and plugins. (#2825)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jnturton
jnturton authored Oct 21, 2023
1 parent 2ab46a9 commit 25d3774
Show file tree
Hide file tree
Showing 20 changed files with 204 additions and 1,858 deletions.
2 changes: 0 additions & 2 deletions common/src/main/java/org/apache/drill/common/config/DrillProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ public final class DrillProperties extends Properties {
public static final String TLS_HANDSHAKE_TIMEOUT = "TLSHandshakeTimeout";
public static final String TLS_PROVIDER = "TLSProvider";
public static final String USE_SYSTEM_TRUSTSTORE = "useSystemTrustStore";
public static final String USE_MAPR_SSL_CONFIG = "useMapRSSLConfig";

public static final String QUERY_TAGS = "queryTags";

// Although all properties from the application are sent to the server (from the client), the following
Expand Down
36 changes: 18 additions & 18 deletions contrib/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,40 +39,40 @@
</dependencies>

<modules>
<!-- Please keep this list sorted. -->
<module>data</module>
<module>udfs</module>
<module>storage-hbase</module>
<module>format-maprdb</module>
<module>format-syslog</module>
<module>format-ltsv</module>
<module>format-access</module>
<module>format-deltalake</module>
<module>format-esri</module>
<module>format-excel</module>
<module>format-hdf5</module>
<module>format-httpd</module>
<module>format-esri</module>
<module>format-iceberg</module>
<module>format-image</module>
<module>format-log</module>
<module>format-access</module>
<module>format-ltsv</module>
<module>format-pcapng</module>
<module>format-pdf</module>
<module>format-hdf5</module>
<module>format-sas</module>
<module>format-spss</module>
<module>format-syslog</module>
<module>format-xml</module>
<module>format-image</module>
<module>format-pcapng</module>
<module>format-iceberg</module>
<module>format-deltalake</module>
<module>storage-cassandra</module>
<module>storage-drill</module>
<module>storage-phoenix</module>
<module>storage-druid</module>
<module>storage-elasticsearch</module>
<module>storage-googlesheets</module>
<module>storage-hbase</module>
<module>storage-hive</module>
<module>storage-mongo</module>
<module>storage-http</module>
<module>storage-jdbc</module>
<module>storage-kafka</module>
<module>storage-kudu</module>
<module>storage-mongo</module>
<module>storage-opentsdb</module>
<module>storage-phoenix</module>
<module>storage-splunk</module>
<module>storage-http</module>
<module>storage-druid</module>
<module>storage-elasticsearch</module>
<module>storage-cassandra</module>
<module>udfs</module>
</modules>

</project>
55 changes: 0 additions & 55 deletions contrib/storage-hbase/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,60 +258,5 @@
</dependency>
</dependencies>
</profile>

<profile>
<id>mapr</id>
<properties>
<alt-hadoop>mapr</alt-hadoop>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
<scope>test</scope>
<classifier>tests</classifier>
<exclusions>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
50 changes: 0 additions & 50 deletions contrib/storage-hive/core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,54 +321,4 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>mapr</id>
<properties>
<freemarker.conf.file>src/main/codegen/config.fmpp</freemarker.conf.file>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-mapr-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>scrMapr/main/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-maprdb-json-handler</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
40 changes: 0 additions & 40 deletions ...b/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
Expand All @@ -50,11 +48,9 @@
import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
import org.apache.drill.exec.store.dfs.FormatPlugin;
import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory;
import com.google.common.collect.ImmutableSet;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.MetaException;
Expand All @@ -64,8 +60,6 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {

private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveStoragePlugin.class);

public static final String HIVE_MAPRDB_FORMAT_PLUGIN_NAME = "hive-maprdb";

private final HiveStoragePluginConfig config;
private HiveSchemaFactory schemaFactory;
private final HiveConf hiveConf;
Expand Down Expand Up @@ -206,44 +200,10 @@ public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext o
options.getBoolean(ExecConstants.HIVE_OPTIMIZE_PARQUET_SCAN_WITH_NATIVE_READER)) {
ruleBuilder.add(ConvertHiveParquetScanToDrillParquetScan.INSTANCE);
}
if (options.getBoolean(ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER)) {
try {
Class<?> hiveToDrillMapRDBJsonRuleClass =
Class.forName("org.apache.drill.exec.planner.sql.logical.ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan");
ruleBuilder.add((StoragePluginOptimizerRule) hiveToDrillMapRDBJsonRuleClass.getField("INSTANCE").get(null));
} catch (ReflectiveOperationException e) {
logger.warn("Current Drill build is not designed for working with Hive MapR-DB tables. " +
"Please disable {} option", ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER);
}
}
return ruleBuilder.build();
}
default:
return ImmutableSet.of();
}
}

@Override
public FormatPlugin getFormatPlugin(FormatPluginConfig formatConfig) {
// TODO: implement formatCreator similar to FileSystemPlugin formatCreator. DRILL-6621
try {
Class<?> mapRDBFormatPluginConfigClass =
Class.forName("org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig");
Class<?> mapRDBFormatPluginClass =
Class.forName("org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin");

if (mapRDBFormatPluginConfigClass.isInstance(formatConfig)) {
return (FormatPlugin) mapRDBFormatPluginClass.getConstructor(
new Class[]{String.class, DrillbitContext.class, Configuration.class,
StoragePluginConfig.class, mapRDBFormatPluginConfigClass})
.newInstance(
new Object[]{HIVE_MAPRDB_FORMAT_PLUGIN_NAME, context, hiveConf, config, formatConfig});
}
} catch (ReflectiveOperationException e) {
throw new DrillRuntimeException("The error is occurred while connecting to MapR-DB or instantiating mapRDBFormatPlugin", e);
}
throw new DrillRuntimeException(String.format("Hive storage plugin doesn't support usage of %s format plugin",
formatConfig.getClass().getName()));
}

}
97 changes: 2 additions & 95 deletions distribution/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,17 +453,6 @@
<artifactId>drill-format-esri</artifactId>
<version>${project.version}</version>
</dependency>
<!--
drill-format-mapr is intentionally excluded from distribution for the default profile
to avoid adding mapr-specific jars into jars/3rdparty folder and inflation of the target jar.
For more details please see DRILL-7213 and DRILL-6929.
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
</dependency>
-->
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-excel</artifactId>
Expand Down Expand Up @@ -499,6 +488,7 @@
<artifactId>drill-storage</artifactId>
<version>${project.version}</version>
</dependency>

</dependencies>
</profile>

Expand Down Expand Up @@ -535,90 +525,6 @@
</plugins>
</build>
</profile>
<profile>
<id>mapr</id>
<dependencies>
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.mapr</groupId>
<artifactId>mapr-test-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-maprdb-json-handler</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.security</groupId>
<artifactId>mapr-security-web</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-winutils</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<id>unpack-winutils</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>process-resources</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-winutils</artifactId>
<version>2.7.0-mapr-1506</version>
<type>zip</type>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/winutils</outputDirectory>
<excludes>**/*.pdb,**/*.lib,**/*.exp</excludes>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rpm</id>
<build>
Expand Down Expand Up @@ -868,6 +774,7 @@
</plugins>
</build>
</profile>

</profiles>

</project>
Loading

0 comments on commit 25d3774

Please sign in to comment.