getViewNames(String schema)
- {
- requireNonNull(schema, "schema is null");
- return metaManager.getViewNames(schema);
- }
-
- public AccumuloView getView(SchemaTableName viewName)
- {
- requireNonNull(viewName, "schema table name is null");
- return metaManager.getView(viewName);
- }
-
- /**
- * Fetches the TabletSplitMetadata for a query against an Accumulo table.
- *
- * Does a whole bunch of fun stuff! Splitting on row ID ranges, applying secondary indexes, column pruning,
- * all sorts of sweet optimizations. What you have here is an important method.
- *
- * @param session Current session
- * @param schema Schema name
- * @param table Table Name
- * @param rowIdDomain Domain for the row ID
- * @param constraints Column constraints for the query
- * @param serializer Instance of a row serializer
- * @return List of TabletSplitMetadata objects for Presto
- */
- public List getTabletSplits(
- ConnectorSession session,
- String schema,
- String table,
- Optional rowIdDomain,
- List constraints,
- AccumuloRowSerializer serializer)
- {
- try {
- String tableName = AccumuloTable.getFullTableName(schema, table);
- LOG.debug("Getting tablet splits for table %s", tableName);
-
- // Get the initial Range based on the row ID domain
- Collection rowIdRanges = getRangesFromDomain(rowIdDomain, serializer);
- List tabletSplits = new ArrayList<>();
-
- // Use the secondary index, if enabled
- if (AccumuloSessionProperties.isOptimizeIndexEnabled(session)) {
- // Get the scan authorizations to query the index
- Authorizations auths = getScanAuthorizations(session, schema, table);
-
- // Check the secondary index based on the column constraints
- // If this returns true, return the tablet splits to Presto
- if (indexLookup.applyIndex(schema, table, session, constraints, rowIdRanges, tabletSplits, serializer, auths)) {
- return tabletSplits;
- }
- }
-
- // If we can't (or shouldn't) use the secondary index, we will just use the Range from the row ID domain
-
- // Split the ranges on tablet boundaries, if enabled
- Collection splitRanges;
- if (AccumuloSessionProperties.isOptimizeSplitRangesEnabled(session)) {
- splitRanges = splitByTabletBoundaries(tableName, rowIdRanges);
- }
- else {
- // if not enabled, just use the same collection
- splitRanges = rowIdRanges;
- }
-
- // Create TabletSplitMetadata objects for each range
- boolean fetchTabletLocations = AccumuloSessionProperties.isOptimizeLocalityEnabled(session);
-
- LOG.debug("Fetching tablet locations: %s", fetchTabletLocations);
-
- for (Range range : splitRanges) {
- // If locality is enabled, then fetch tablet location
- if (fetchTabletLocations) {
- tabletSplits.add(new TabletSplitMetadata(getTabletLocation(tableName, range.getStartKey()), ImmutableList.of(range)));
- }
- else {
- // else, just use the default location
- tabletSplits.add(new TabletSplitMetadata(Optional.empty(), ImmutableList.of(range)));
- }
- }
-
- // Log some fun stuff and return the tablet splits
- LOG.debug("Number of splits for table %s is %d with %d ranges", tableName, tabletSplits.size(), splitRanges.size());
- return tabletSplits;
- }
- catch (Exception e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get splits from Accumulo", e);
- }
- }
-
- /**
- * Gets the scan authorizations to use for scanning tables.
- *
- * In order of priority: session username authorizations, then table property, then the default connector auths.
- *
- * @param session Current session
- * @param schema Schema name
- * @param table Table Name
- * @return Scan authorizations
- * @throws AccumuloException If a generic Accumulo error occurs
- * @throws AccumuloSecurityException If a security exception occurs
- */
- private Authorizations getScanAuthorizations(ConnectorSession session, String schema,
- String table)
- throws AccumuloException, AccumuloSecurityException
- {
- String sessionScanUser = AccumuloSessionProperties.getScanUsername(session);
- if (sessionScanUser != null) {
- Authorizations scanAuths = connector.securityOperations().getUserAuthorizations(sessionScanUser);
- LOG.debug("Using session scan auths for user %s: %s", sessionScanUser, scanAuths);
- return scanAuths;
- }
-
- AccumuloTable accumuloTable = this.getTable(new SchemaTableName(schema, table));
- if (accumuloTable == null) {
- throw new TableNotFoundException(new SchemaTableName(schema, table));
- }
-
- Optional strAuths = accumuloTable.getScanAuthorizations();
- if (strAuths.isPresent()) {
- Authorizations scanAuths = new Authorizations(Iterables.toArray(COMMA_SPLITTER.split(strAuths.get()), String.class));
- LOG.debug("scan_auths table property set, using: %s", scanAuths);
- return scanAuths;
- }
-
- LOG.debug("scan_auths table property not set, using connector auths: %s", this.auths);
- return this.auths;
- }
-
- private Collection splitByTabletBoundaries(String tableName, Collection ranges)
- throws org.apache.accumulo.core.client.TableNotFoundException, AccumuloException, AccumuloSecurityException
- {
- ImmutableSet.Builder rangeBuilder = ImmutableSet.builder();
- for (Range range : ranges) {
- // if start and end key are equivalent, no need to split the range
- if (range.getStartKey() != null && range.getEndKey() != null && range.getStartKey().equals(range.getEndKey())) {
- rangeBuilder.add(range);
- }
- else {
- // Call out to Accumulo to split the range on tablets
- rangeBuilder.addAll(connector.tableOperations().splitRangeByTablets(tableName, range, Integer.MAX_VALUE));
- }
- }
- return rangeBuilder.build();
- }
-
- /**
- * Gets the TabletServer hostname for where the given key is located in the given table
- *
- * @param table Fully-qualified table name
- * @param key Key to locate
- * @return The tablet location, or DUMMY_LOCATION if an error occurs
- */
- private Optional getTabletLocation(String table, Key key)
- {
- try {
- // Get the Accumulo table ID so we can scan some fun stuff
- String tableId = connector.tableOperations().tableIdMap().get(table);
-
- // Create our scanner against the metadata table, fetching 'loc' family
- Scanner scanner = connector.createScanner("accumulo.metadata", auths);
- scanner.fetchColumnFamily(new Text("loc"));
-
- // Set the scan range to just this table, from the table ID to the default tablet
- // row, which is the last listed tablet
- Key defaultTabletRow = new Key(tableId + '<');
- Key start = new Key(tableId);
- Key end = defaultTabletRow.followingKey(PartialKey.ROW);
- scanner.setRange(new Range(start, end));
-
- Optional location = Optional.empty();
- if (key == null) {
- // if the key is null, then it is -inf, so get first tablet location
- Iterator> iter = scanner.iterator();
- if (iter.hasNext()) {
- location = Optional.of(iter.next().getValue().toString());
- }
- }
- else {
- // Else, we will need to scan through the tablet location data and find the location
-
- // Create some text objects to do comparison for what we are looking for
- Text splitCompareKey = new Text();
- key.getRow(splitCompareKey);
- Text scannedCompareKey = new Text();
-
- // Scan the table!
- for (Entry entry : scanner) {
- // Get the bytes of the key
- byte[] keyBytes = entry.getKey().getRow().copyBytes();
-
- // If the last byte is <, then we have hit the default tablet, so use this location
- if (keyBytes[keyBytes.length - 1] == '<') {
- location = Optional.of(entry.getValue().toString());
- break;
- }
- else {
- // Chop off some magic nonsense
- scannedCompareKey.set(keyBytes, 3, keyBytes.length - 3);
-
- // Compare the keys, moving along the tablets until the location is found
- if (scannedCompareKey.getLength() > 0) {
- int compareTo = splitCompareKey.compareTo(scannedCompareKey);
- if (compareTo <= 0) {
- location = Optional.of(entry.getValue().toString());
- }
- else {
- // all future tablets will be greater than this key
- break;
- }
- }
- }
- }
- scanner.close();
- }
-
- // If we were unable to find the location for some reason, return the default tablet
- // location
- return location.isPresent() ? location : getDefaultTabletLocation(table);
- }
- catch (Exception e) {
- // Swallow this exception so the query does not fail due to being unable
- // to locate the tablet server for the provided Key.
- // This is purely an optimization, but we will want to log the error.
- LOG.error("Failed to get tablet location, returning dummy location", e);
- return Optional.empty();
- }
- }
-
- private Optional getDefaultTabletLocation(String fulltable)
- {
- try {
- String tableId = connector.tableOperations().tableIdMap().get(fulltable);
-
- // Create a scanner over the metadata table, fetching the 'loc' column of the default tablet row
- Scanner scan = connector.createScanner("accumulo.metadata", connector.securityOperations().getUserAuthorizations(username));
- scan.fetchColumnFamily(new Text("loc"));
- scan.setRange(new Range(tableId + '<'));
-
- // scan the entry
- Optional location = Optional.empty();
- for (Entry entry : scan) {
- if (location.isPresent()) {
- throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Scan for default tablet returned more than one entry");
- }
-
- location = Optional.of(entry.getValue().toString());
- }
-
- scan.close();
- return location;
- }
- catch (Exception e) {
- // Swallow this exception so the query does not fail due to being unable to locate the tablet server for the default tablet.
- // This is purely an optimization, but we will want to log the error.
- LOG.error("Failed to get tablet location, returning dummy location", e);
- return Optional.empty();
- }
- }
-
- /**
- * Gets a collection of Accumulo Range objects from the given Presto domain.
- * This maps the column constraints of the given Domain to an Accumulo Range scan.
- *
- * @param domain Domain, can be null (returns (-inf, +inf) Range)
- * @param serializer Instance of an {@link AccumuloRowSerializer}
- * @return A collection of Accumulo Range objects
- * @throws TableNotFoundException If the Accumulo table is not found
- */
- public static Collection getRangesFromDomain(Optional domain, AccumuloRowSerializer serializer)
- throws TableNotFoundException
- {
- // if we have no predicate pushdown, use the full range
- if (!domain.isPresent()) {
- return ImmutableSet.of(new Range());
- }
-
- ImmutableSet.Builder rangeBuilder = ImmutableSet.builder();
- for (io.prestosql.spi.predicate.Range range : domain.get().getValues().getRanges().getOrderedRanges()) {
- rangeBuilder.add(getRangeFromPrestoRange(range, serializer));
- }
-
- return rangeBuilder.build();
- }
-
- private static Range getRangeFromPrestoRange(io.prestosql.spi.predicate.Range prestoRange, AccumuloRowSerializer serializer)
- throws TableNotFoundException
- {
- Range accumuloRange;
- if (prestoRange.isAll()) {
- accumuloRange = new Range();
- }
- else if (prestoRange.isSingleValue()) {
- Text split = new Text(serializer.encode(prestoRange.getType(), prestoRange.getSingleValue()));
- accumuloRange = new Range(split);
- }
- else {
- if (prestoRange.getLow().isLowerUnbounded()) {
- // If low is unbounded, then create a range from (-inf, value), checking inclusivity
- boolean inclusive = prestoRange.getHigh().getBound() == Bound.EXACTLY;
- Text split = new Text(serializer.encode(prestoRange.getType(), prestoRange.getHigh().getValue()));
- accumuloRange = new Range(null, false, split, inclusive);
- }
- else if (prestoRange.getHigh().isUpperUnbounded()) {
- // If high is unbounded, then create a range from (value, +inf), checking inclusivity
- boolean inclusive = prestoRange.getLow().getBound() == Bound.EXACTLY;
- Text split = new Text(serializer.encode(prestoRange.getType(), prestoRange.getLow().getValue()));
- accumuloRange = new Range(split, inclusive, null, false);
- }
- else {
- // If high is unbounded, then create a range from low to high, checking inclusivity
- boolean startKeyInclusive = prestoRange.getLow().getBound() == Bound.EXACTLY;
- Text startSplit = new Text(serializer.encode(prestoRange.getType(), prestoRange.getLow().getValue()));
-
- boolean endKeyInclusive = prestoRange.getHigh().getBound() == Bound.EXACTLY;
- Text endSplit = new Text(serializer.encode(prestoRange.getType(), prestoRange.getHigh().getValue()));
- accumuloRange = new Range(startSplit, startKeyInclusive, endSplit, endKeyInclusive);
- }
- }
-
- return accumuloRange;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnector.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnector.java
deleted file mode 100644
index 74bd9c957921..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnector.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import io.airlift.bootstrap.LifeCycleManager;
-import io.airlift.log.Logger;
-import io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties;
-import io.prestosql.plugin.accumulo.conf.AccumuloTableProperties;
-import io.prestosql.plugin.accumulo.io.AccumuloPageSinkProvider;
-import io.prestosql.plugin.accumulo.io.AccumuloRecordSetProvider;
-import io.prestosql.spi.connector.Connector;
-import io.prestosql.spi.connector.ConnectorMetadata;
-import io.prestosql.spi.connector.ConnectorPageSinkProvider;
-import io.prestosql.spi.connector.ConnectorRecordSetProvider;
-import io.prestosql.spi.connector.ConnectorSplitManager;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-import io.prestosql.spi.session.PropertyMetadata;
-import io.prestosql.spi.transaction.IsolationLevel;
-
-import javax.inject.Inject;
-
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static io.prestosql.spi.transaction.IsolationLevel.READ_UNCOMMITTED;
-import static io.prestosql.spi.transaction.IsolationLevel.checkConnectorSupports;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Presto Connector for Accumulo.
- * Defines several high-level classes for properties, metadata, retrieving splits, providing I/O operations, etc.
- */
-public class AccumuloConnector
- implements Connector
-{
- private static final Logger LOG = Logger.get(AccumuloConnector.class);
-
- private final LifeCycleManager lifeCycleManager;
- private final AccumuloMetadataFactory metadataFactory;
- private final AccumuloSplitManager splitManager;
- private final AccumuloRecordSetProvider recordSetProvider;
- private final AccumuloPageSinkProvider pageSinkProvider;
- private final AccumuloSessionProperties sessionProperties;
- private final AccumuloTableProperties tableProperties;
- private final ConcurrentMap transactions = new ConcurrentHashMap<>();
-
- @Inject
- public AccumuloConnector(
- LifeCycleManager lifeCycleManager,
- AccumuloMetadataFactory metadataFactory,
- AccumuloSplitManager splitManager,
- AccumuloRecordSetProvider recordSetProvider,
- AccumuloPageSinkProvider pageSinkProvider,
- AccumuloSessionProperties sessionProperties,
- AccumuloTableProperties tableProperties)
- {
- this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null");
- this.metadataFactory = requireNonNull(metadataFactory, "metadata is null");
- this.splitManager = requireNonNull(splitManager, "splitManager is null");
- this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null");
- this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null");
- this.sessionProperties = requireNonNull(sessionProperties, "sessionProperties is null");
- this.tableProperties = requireNonNull(tableProperties, "tableProperties is null");
- }
-
- @Override
- public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle)
- {
- ConnectorMetadata metadata = transactions.get(transactionHandle);
- checkArgument(metadata != null, "no such transaction: %s", transactionHandle);
- return metadata;
- }
-
- @Override
- public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly)
- {
- checkConnectorSupports(READ_UNCOMMITTED, isolationLevel);
- ConnectorTransactionHandle transaction = new AccumuloTransactionHandle();
- transactions.put(transaction, metadataFactory.create());
- return transaction;
- }
-
- @Override
- public void commit(ConnectorTransactionHandle transactionHandle)
- {
- checkArgument(transactions.remove(transactionHandle) != null, "no such transaction: %s", transactionHandle);
- }
-
- @Override
- public void rollback(ConnectorTransactionHandle transactionHandle)
- {
- AccumuloMetadata metadata = transactions.remove(transactionHandle);
- checkArgument(metadata != null, "no such transaction: %s", transactionHandle);
- metadata.rollback();
- }
-
- @Override
- public ConnectorSplitManager getSplitManager()
- {
- return splitManager;
- }
-
- @Override
- public ConnectorRecordSetProvider getRecordSetProvider()
- {
- return recordSetProvider;
- }
-
- @Override
- public ConnectorPageSinkProvider getPageSinkProvider()
- {
- return pageSinkProvider;
- }
-
- @Override
- public List> getTableProperties()
- {
- return tableProperties.getTableProperties();
- }
-
- @Override
- public List> getSessionProperties()
- {
- return sessionProperties.getSessionProperties();
- }
-
- @Override
- public final void shutdown()
- {
- lifeCycleManager.stop();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnectorFactory.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnectorFactory.java
deleted file mode 100644
index 624daec9a183..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloConnectorFactory.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.google.inject.Injector;
-import io.airlift.bootstrap.Bootstrap;
-import io.airlift.json.JsonModule;
-import io.prestosql.spi.connector.Connector;
-import io.prestosql.spi.connector.ConnectorContext;
-import io.prestosql.spi.connector.ConnectorFactory;
-import io.prestosql.spi.connector.ConnectorHandleResolver;
-
-import java.util.Map;
-
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloConnectorFactory
- implements ConnectorFactory
-{
- public static final String CONNECTOR_NAME = "accumulo";
-
- @Override
- public String getName()
- {
- return CONNECTOR_NAME;
- }
-
- @Override
- public Connector create(String catalogName, Map config, ConnectorContext context)
- {
- requireNonNull(catalogName, "catalogName is null");
- requireNonNull(config, "requiredConfig is null");
- requireNonNull(context, "context is null");
-
- Bootstrap app = new Bootstrap(
- new JsonModule(),
- new AccumuloModule(context.getTypeManager()));
-
- Injector injector = app
- .strictConfig()
- .doNotInitializeLogging()
- .setRequiredConfigurationProperties(config)
- .initialize();
-
- return injector.getInstance(AccumuloConnector.class);
- }
-
- @Override
- public ConnectorHandleResolver getHandleResolver()
- {
- return new AccumuloHandleResolver();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloErrorCode.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloErrorCode.java
deleted file mode 100644
index 45b0329196ad..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloErrorCode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import io.prestosql.spi.ErrorCode;
-import io.prestosql.spi.ErrorCodeSupplier;
-import io.prestosql.spi.ErrorType;
-
-import static io.prestosql.spi.ErrorType.EXTERNAL;
-
-public enum AccumuloErrorCode
- implements ErrorCodeSupplier
-{
- // Thrown when an Accumulo error is caught that we were not expecting,
- // such as when a create table operation fails (even though we know it will succeed due to our validation steps)
- UNEXPECTED_ACCUMULO_ERROR(1, EXTERNAL),
-
- // Thrown when a ZooKeeper error is caught due to a failed operation
- ZOOKEEPER_ERROR(2, EXTERNAL),
-
- // Thrown when a serialization error occurs when reading/writing data from/to Accumulo
- IO_ERROR(3, EXTERNAL),
-
- // Thrown when a table that is expected to exist does not exist
- ACCUMULO_TABLE_DNE(4, EXTERNAL),
-
- // Thrown when a table that is *not* expected to exist, does exist
- ACCUMULO_TABLE_EXISTS(5, EXTERNAL),
-
- // Thrown when an attempt to start/stop MiniAccumuloCluster fails (testing only)
- MINI_ACCUMULO(6, EXTERNAL);
-
- private final ErrorCode errorCode;
-
- AccumuloErrorCode(int code, ErrorType type)
- {
- errorCode = new ErrorCode(code + 0x0103_0000, name(), type);
- }
-
- @Override
- public ErrorCode toErrorCode()
- {
- return errorCode;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloHandleResolver.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloHandleResolver.java
deleted file mode 100644
index 5479eb84742e..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloHandleResolver.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.AccumuloSplit;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ConnectorHandleResolver;
-import io.prestosql.spi.connector.ConnectorInsertTableHandle;
-import io.prestosql.spi.connector.ConnectorOutputTableHandle;
-import io.prestosql.spi.connector.ConnectorSplit;
-import io.prestosql.spi.connector.ConnectorTableHandle;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-
-public class AccumuloHandleResolver
- implements ConnectorHandleResolver
-{
- @Override
- public Class getTableHandleClass()
- {
- return AccumuloTableHandle.class;
- }
-
- @Override
- public Class getInsertTableHandleClass()
- {
- return AccumuloTableHandle.class;
- }
-
- @Override
- public Class getOutputTableHandleClass()
- {
- return AccumuloTableHandle.class;
- }
-
- @Override
- public Class getColumnHandleClass()
- {
- return AccumuloColumnHandle.class;
- }
-
- @Override
- public Class getSplitClass()
- {
- return AccumuloSplit.class;
- }
-
- @Override
- public Class getTransactionHandleClass()
- {
- return AccumuloTransactionHandle.class;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadata.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadata.java
deleted file mode 100644
index 3311f0868690..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadata.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.airlift.json.JsonCodec;
-import io.airlift.json.JsonCodecFactory;
-import io.airlift.json.ObjectMapperProvider;
-import io.airlift.slice.Slice;
-import io.prestosql.plugin.accumulo.metadata.AccumuloTable;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ColumnMetadata;
-import io.prestosql.spi.connector.ConnectorInsertTableHandle;
-import io.prestosql.spi.connector.ConnectorMetadata;
-import io.prestosql.spi.connector.ConnectorNewTableLayout;
-import io.prestosql.spi.connector.ConnectorOutputMetadata;
-import io.prestosql.spi.connector.ConnectorOutputTableHandle;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorTableHandle;
-import io.prestosql.spi.connector.ConnectorTableMetadata;
-import io.prestosql.spi.connector.ConnectorTableProperties;
-import io.prestosql.spi.connector.ConnectorViewDefinition;
-import io.prestosql.spi.connector.Constraint;
-import io.prestosql.spi.connector.ConstraintApplicationResult;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.connector.SchemaTablePrefix;
-import io.prestosql.spi.connector.TableNotFoundException;
-import io.prestosql.spi.predicate.TupleDomain;
-import io.prestosql.spi.statistics.ComputedStatistics;
-
-import javax.inject.Inject;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static com.google.common.base.Preconditions.checkState;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_EXISTS;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
-import static java.lang.String.format;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Presto metadata provider for Accumulo.
- * Responsible for creating/dropping/listing tables, schemas, columns, all sorts of goodness. Heavily leverages {@link AccumuloClient}.
- */
-public class AccumuloMetadata
- implements ConnectorMetadata
-{
- private static final JsonCodec VIEW_CODEC =
- new JsonCodecFactory(new ObjectMapperProvider()).jsonCodec(ConnectorViewDefinition.class);
-
- private final AccumuloClient client;
- private final AtomicReference rollbackAction = new AtomicReference<>();
-
- @Inject
- public AccumuloMetadata(AccumuloClient client)
- {
- this.client = requireNonNull(client, "client is null");
- }
-
- @Override
- public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout)
- {
- checkNoRollback();
-
- SchemaTableName tableName = tableMetadata.getTable();
- AccumuloTable table = client.createTable(tableMetadata);
-
- AccumuloTableHandle handle = new AccumuloTableHandle(
- tableName.getSchemaName(),
- tableName.getTableName(),
- table.getRowId(),
- table.isExternal(),
- table.getSerializerClassName(),
- table.getScanAuthorizations());
-
- setRollback(() -> rollbackCreateTable(table));
-
- return handle;
- }
-
- @Override
- public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics)
- {
- clearRollback();
- return Optional.empty();
- }
-
- private void rollbackCreateTable(AccumuloTable table)
- {
- client.dropTable(table);
- }
-
- @Override
- public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
- {
- client.createTable(tableMetadata);
- }
-
- @Override
- public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
- AccumuloTable table = client.getTable(handle.toSchemaTableName());
- if (table != null) {
- client.dropTable(table);
- }
- }
-
- @Override
- public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle,
- SchemaTableName newTableName)
- {
- if (client.getTable(newTableName) != null) {
- throw new PrestoException(ACCUMULO_TABLE_EXISTS, "Table " + newTableName + " already exists");
- }
-
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
- client.renameTable(handle.toSchemaTableName(), newTableName);
- }
-
- @Override
- public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, boolean replace)
- {
- String viewData = VIEW_CODEC.toJson(definition);
- if (replace) {
- client.createOrReplaceView(viewName, viewData);
- }
- else {
- client.createView(viewName, viewData);
- }
- }
-
- @Override
- public void dropView(ConnectorSession session, SchemaTableName viewName)
- {
- client.dropView(viewName);
- }
-
- @Override
- public Optional getView(ConnectorSession session, SchemaTableName viewName)
- {
- return Optional.ofNullable(client.getView(viewName))
- .map(view -> VIEW_CODEC.fromJson(view.getData()));
- }
-
- @Override
- public List listViews(ConnectorSession session, Optional schemaName)
- {
- return listViews(schemaName);
- }
-
- /**
- * Gets all views in the given schema, or all schemas if null.
- *
- * @param filterSchema Schema to filter the views, or absent to list all schemas
- * @return List of views
- */
- private List listViews(Optional filterSchema)
- {
- ImmutableList.Builder builder = ImmutableList.builder();
- if (filterSchema.isPresent()) {
- for (String view : client.getViewNames(filterSchema.get())) {
- builder.add(new SchemaTableName(filterSchema.get(), view));
- }
- }
- else {
- for (String schemaName : client.getSchemaNames()) {
- for (String view : client.getViewNames(schemaName)) {
- builder.add(new SchemaTableName(schemaName, view));
- }
- }
- }
-
- return builder.build();
- }
-
- @Override
- public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle)
- {
- checkNoRollback();
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
- setRollback(() -> rollbackInsert(handle));
- return handle;
- }
-
- @Override
- public Optional finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection fragments, Collection computedStatistics)
- {
- clearRollback();
- return Optional.empty();
- }
-
- private static void rollbackInsert(ConnectorInsertTableHandle insertHandle)
- {
- // Rollbacks for inserts are off the table when it comes to data in Accumulo.
- // When a batch of Mutations fails to be inserted, the general strategy
- // is to run the insert operation again until it is successful
- // Any mutations that were successfully written will be overwritten
- // with the same values, so that isn't a problem.
- AccumuloTableHandle handle = (AccumuloTableHandle) insertHandle;
- throw new PrestoException(NOT_SUPPORTED, format("Unable to rollback insert for table %s.%s. Some rows may have been written. Please run your insert again.", handle.getSchema(), handle.getTable()));
- }
-
- @Override
- public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
- {
- if (!listSchemaNames(session).contains(tableName.getSchemaName().toLowerCase(Locale.ENGLISH))) {
- return null;
- }
-
- // Need to validate that SchemaTableName is a table
- if (!this.listViews(session, Optional.of(tableName.getSchemaName())).contains(tableName)) {
- AccumuloTable table = client.getTable(tableName);
- if (table == null) {
- return null;
- }
-
- return new AccumuloTableHandle(
- table.getSchema(),
- table.getTable(),
- table.getRowId(),
- table.isExternal(),
- table.getSerializerClassName(),
- table.getScanAuthorizations());
- }
-
- return null;
- }
-
- @Override
- public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) table;
- SchemaTableName tableName = new SchemaTableName(handle.getSchema(), handle.getTable());
- ConnectorTableMetadata metadata = getTableMetadata(tableName);
- if (metadata == null) {
- throw new TableNotFoundException(tableName);
- }
- return metadata;
- }
-
- @Override
- public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
-
- AccumuloTable table = client.getTable(handle.toSchemaTableName());
- if (table == null) {
- throw new TableNotFoundException(handle.toSchemaTableName());
- }
-
- ImmutableMap.Builder columnHandles = ImmutableMap.builder();
- for (AccumuloColumnHandle column : table.getColumns()) {
- columnHandles.put(column.getName(), column);
- }
- return columnHandles.build();
- }
-
- @Override
- public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle)
- {
- return ((AccumuloColumnHandle) columnHandle).getColumnMetadata();
- }
-
- @Override
- public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
- AccumuloColumnHandle columnHandle = (AccumuloColumnHandle) source;
- AccumuloTable table = client.getTable(handle.toSchemaTableName());
- if (table == null) {
- throw new TableNotFoundException(new SchemaTableName(handle.getSchema(), handle.getTable()));
- }
-
- client.renameColumn(table, columnHandle.getName(), target);
- }
-
- @Override
- public List listSchemaNames(ConnectorSession session)
- {
- return ImmutableList.copyOf(client.getSchemaNames());
- }
-
- @Override
- public List listTables(ConnectorSession session, Optional filterSchema)
- {
- Set schemaNames = filterSchema.>map(ImmutableSet::of)
- .orElseGet(client::getSchemaNames);
-
- ImmutableList.Builder builder = ImmutableList.builder();
- for (String schemaName : schemaNames) {
- for (String tableName : client.getTableNames(schemaName)) {
- builder.add(new SchemaTableName(schemaName, tableName));
- }
- }
- return builder.build();
- }
-
- @Override
- public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
- {
- requireNonNull(prefix, "prefix is null");
- ImmutableMap.Builder> columns = ImmutableMap.builder();
- for (SchemaTableName tableName : listTables(session, prefix)) {
- ConnectorTableMetadata tableMetadata = getTableMetadata(tableName);
- // table can disappear during listing operation
- if (tableMetadata != null) {
- columns.put(tableName, tableMetadata.getColumns());
- }
- }
- return columns.build();
- }
-
- @Override
- public boolean usesLegacyTableLayouts()
- {
- return false;
- }
-
- @Override
- public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle table, Constraint constraint)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) table;
-
- TupleDomain oldDomain = handle.getConstraint();
- TupleDomain newDomain = oldDomain.intersect(constraint.getSummary());
- if (oldDomain.equals(newDomain)) {
- return Optional.empty();
- }
-
- handle = new AccumuloTableHandle(
- handle.getSchema(),
- handle.getTable(),
- handle.getRowId(),
- newDomain,
- handle.isExternal(),
- handle.getSerializerClassName(),
- handle.getScanAuthorizations());
-
- return Optional.of(new ConstraintApplicationResult<>(handle, constraint.getSummary()));
- }
-
- @Override
- public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle handle)
- {
- return new ConnectorTableProperties();
- }
-
- private void checkNoRollback()
- {
- checkState(rollbackAction.get() == null, "Cannot begin a new write while in an existing one");
- }
-
- private void setRollback(Runnable action)
- {
- checkState(rollbackAction.compareAndSet(null, action), "Should not have to override existing rollback action");
- }
-
- private void clearRollback()
- {
- rollbackAction.set(null);
- }
-
- public void rollback()
- {
- Runnable rollbackAction = this.rollbackAction.getAndSet(null);
- if (rollbackAction != null) {
- rollbackAction.run();
- }
- }
-
- private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName)
- {
- if (!client.getSchemaNames().contains(tableName.getSchemaName())) {
- return null;
- }
-
- // Need to validate that SchemaTableName is a table
- if (!this.listViews(Optional.ofNullable(tableName.getSchemaName())).contains(tableName)) {
- AccumuloTable table = client.getTable(tableName);
- if (table == null) {
- return null;
- }
-
- return new ConnectorTableMetadata(tableName, table.getColumnsMetadata());
- }
-
- return null;
- }
-
- private List listTables(ConnectorSession session, SchemaTablePrefix prefix)
- {
- // List all tables if schema or table is null
- if (!prefix.getTable().isPresent()) {
- return listTables(session, prefix.getSchema());
- }
-
- // Make sure requested table exists, returning the single table of it does
- SchemaTableName table = prefix.toSchemaTableName();
- if (getTableHandle(session, table) != null) {
- return ImmutableList.of(table);
- }
-
- // Else, return empty list
- return ImmutableList.of();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadataFactory.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadataFactory.java
deleted file mode 100644
index d5f8bf27ba0d..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloMetadataFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import javax.inject.Inject;
-
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloMetadataFactory
-{
- private final AccumuloClient client;
-
- @Inject
- public AccumuloMetadataFactory(AccumuloClient client)
- {
- this.client = requireNonNull(client, "client is null");
- }
-
- public AccumuloMetadata create()
- {
- return new AccumuloMetadata(client);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloModule.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloModule.java
deleted file mode 100644
index a341ef934a35..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloModule.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
-import com.google.inject.Binder;
-import com.google.inject.Module;
-import com.google.inject.Scopes;
-import io.airlift.json.JsonCodec;
-import io.airlift.log.Logger;
-import io.prestosql.plugin.accumulo.conf.AccumuloConfig;
-import io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties;
-import io.prestosql.plugin.accumulo.conf.AccumuloTableProperties;
-import io.prestosql.plugin.accumulo.index.ColumnCardinalityCache;
-import io.prestosql.plugin.accumulo.index.IndexLookup;
-import io.prestosql.plugin.accumulo.io.AccumuloPageSinkProvider;
-import io.prestosql.plugin.accumulo.io.AccumuloRecordSetProvider;
-import io.prestosql.plugin.accumulo.metadata.AccumuloTable;
-import io.prestosql.plugin.accumulo.metadata.ZooKeeperMetadataManager;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeId;
-import io.prestosql.spi.type.TypeManager;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.log4j.JulAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.PatternLayout;
-
-import javax.inject.Inject;
-import javax.inject.Provider;
-
-import static io.airlift.configuration.ConfigBinder.configBinder;
-import static io.airlift.json.JsonBinder.jsonBinder;
-import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Presto module to do all kinds of run Guice injection stuff!
- *
- * WARNING: Contains black magick
- */
-public class AccumuloModule
- implements Module
-{
- private final TypeManager typeManager;
-
- public AccumuloModule(TypeManager typeManager)
- {
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
- }
-
- @Override
- public void configure(Binder binder)
- {
- // Add appender to Log4J root logger
- JulAppender appender = new JulAppender(); //create appender
- appender.setLayout(new PatternLayout("%d %-5p %c - %m%n"));
- appender.setThreshold(Level.INFO);
- appender.activateOptions();
- org.apache.log4j.Logger.getRootLogger().addAppender(appender);
-
- binder.bind(TypeManager.class).toInstance(typeManager);
-
- binder.bind(AccumuloConnector.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloMetadata.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloMetadataFactory.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloClient.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloSplitManager.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloRecordSetProvider.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloPageSinkProvider.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloHandleResolver.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloSessionProperties.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloTableProperties.class).in(Scopes.SINGLETON);
- binder.bind(ZooKeeperMetadataManager.class).in(Scopes.SINGLETON);
- binder.bind(AccumuloTableManager.class).in(Scopes.SINGLETON);
- binder.bind(IndexLookup.class).in(Scopes.SINGLETON);
- binder.bind(ColumnCardinalityCache.class).in(Scopes.SINGLETON);
- binder.bind(Connector.class).toProvider(ConnectorProvider.class);
-
- configBinder(binder).bindConfig(AccumuloConfig.class);
-
- jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
- jsonCodecBinder(binder).bindMapJsonCodec(String.class, JsonCodec.listJsonCodec(AccumuloTable.class));
- }
-
- public static final class TypeDeserializer
- extends FromStringDeserializer
- {
- private final TypeManager typeManager;
-
- @Inject
- public TypeDeserializer(TypeManager typeManager)
- {
- super(Type.class);
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
- }
-
- @Override
- protected Type _deserialize(String value, DeserializationContext context)
- {
- return typeManager.getType(TypeId.of(value));
- }
- }
-
- private static class ConnectorProvider
- implements Provider
- {
- private static final Logger LOG = Logger.get(ConnectorProvider.class);
-
- private final String instance;
- private final String zooKeepers;
- private final String username;
- private final String password;
-
- @Inject
- public ConnectorProvider(AccumuloConfig config)
- {
- requireNonNull(config, "config is null");
- this.instance = config.getInstance();
- this.zooKeepers = config.getZooKeepers();
- this.username = config.getUsername();
- this.password = config.getPassword();
- }
-
- @Override
- public Connector get()
- {
- try {
- Instance inst = new ZooKeeperInstance(instance, zooKeepers);
- Connector connector = inst.getConnector(username, new PasswordToken(password.getBytes(UTF_8)));
- LOG.info("Connection to instance %s at %s established, user %s", instance, zooKeepers, username);
- return connector;
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
- }
- }
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloPlugin.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloPlugin.java
deleted file mode 100644
index 888ab9cb7f34..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloPlugin.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.google.common.collect.ImmutableList;
-import io.prestosql.spi.Plugin;
-import io.prestosql.spi.connector.ConnectorFactory;
-
-public class AccumuloPlugin
- implements Plugin
-{
- @Override
- public Iterable getConnectorFactories()
- {
- return ImmutableList.of(new AccumuloConnectorFactory());
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloSplitManager.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloSplitManager.java
deleted file mode 100644
index 196ce3745236..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloSplitManager.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.google.common.collect.ImmutableList;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnConstraint;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.AccumuloSplit;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.plugin.accumulo.model.TabletSplitMetadata;
-import io.prestosql.plugin.accumulo.model.WrappedRange;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorSplit;
-import io.prestosql.spi.connector.ConnectorSplitManager;
-import io.prestosql.spi.connector.ConnectorSplitSource;
-import io.prestosql.spi.connector.ConnectorTableHandle;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-import io.prestosql.spi.connector.FixedSplitSource;
-import io.prestosql.spi.predicate.Domain;
-import io.prestosql.spi.predicate.TupleDomain;
-import io.prestosql.spi.predicate.TupleDomain.ColumnDomain;
-
-import javax.inject.Inject;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloSplitManager
- implements ConnectorSplitManager
-{
- private final AccumuloClient client;
-
- @Inject
- public AccumuloSplitManager(AccumuloClient client)
- {
- this.client = requireNonNull(client, "client is null");
- }
-
- @Override
- public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle tableHandle, SplitSchedulingStrategy splitSchedulingStrategy)
- {
- AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle;
-
- String schemaName = handle.getSchema();
- String tableName = handle.getTable();
- String rowIdName = handle.getRowId();
-
- // Get non-row ID column constraints
- List constraints = getColumnConstraints(rowIdName, handle.getConstraint());
-
- // Get the row domain column range
- Optional rDom = getRangeDomain(rowIdName, handle.getConstraint());
-
- // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index
- List tabletSplits = client.getTabletSplits(session, schemaName, tableName, rDom, constraints, handle.getSerializerInstance());
-
- // Pack the tablet split metadata into a connector split
- ImmutableList.Builder cSplits = ImmutableList.builder();
- for (TabletSplitMetadata splitMetadata : tabletSplits) {
- AccumuloSplit split = new AccumuloSplit(
- splitMetadata.getRanges().stream().map(WrappedRange::new).collect(Collectors.toList()),
- splitMetadata.getHostPort());
- cSplits.add(split);
- }
-
- return new FixedSplitSource(cSplits.build());
- }
-
- private static Optional getRangeDomain(String rowIdName, TupleDomain constraint)
- {
- if (constraint.getColumnDomains().isPresent()) {
- for (ColumnDomain cd : constraint.getColumnDomains().get()) {
- AccumuloColumnHandle col = (AccumuloColumnHandle) cd.getColumn();
- if (col.getName().equals(rowIdName)) {
- return Optional.of(cd.getDomain());
- }
- }
- }
-
- return Optional.empty();
- }
-
- /**
- * Gets a list of {@link AccumuloColumnConstraint} based on the given constraint ID, excluding the row ID column
- *
- * @param rowIdName Presto column name mapping to the Accumulo row ID
- * @param constraint Set of query constraints
- * @return List of all column constraints
- */
- private static List getColumnConstraints(String rowIdName, TupleDomain constraint)
- {
- ImmutableList.Builder constraintBuilder = ImmutableList.builder();
- for (ColumnDomain columnDomain : constraint.getColumnDomains().get()) {
- AccumuloColumnHandle columnHandle = (AccumuloColumnHandle) columnDomain.getColumn();
-
- if (!columnHandle.getName().equals(rowIdName)) {
- // Family and qualifier will exist for non-row ID columns
- constraintBuilder.add(new AccumuloColumnConstraint(
- columnHandle.getName(),
- columnHandle.getFamily().get(),
- columnHandle.getQualifier().get(),
- Optional.of(columnDomain.getDomain()),
- columnHandle.isIndexed()));
- }
- }
-
- return constraintBuilder.build();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTableManager.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTableManager.java
deleted file mode 100644
index 7b2f2f792d02..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTableManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import io.airlift.log.Logger;
-import io.prestosql.spi.PrestoException;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.NamespaceExistsException;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.hadoop.io.Text;
-
-import javax.inject.Inject;
-
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_EXISTS;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static java.util.Objects.requireNonNull;
-
-/**
- * This class is a light wrapper for Accumulo's Connector object.
- * It will perform the given operation, or throw an exception if an Accumulo- or ZooKeeper-based error occurs.
- */
-public class AccumuloTableManager
-{
- private static final Logger LOG = Logger.get(AccumuloTableManager.class);
- private static final String DEFAULT = "default";
- private final Connector connector;
-
- @Inject
- public AccumuloTableManager(Connector connector)
- {
- this.connector = requireNonNull(connector, "connector is null");
- }
-
- /**
- * Ensures the given Accumulo namespace exist, creating it if necessary
- *
- * @param schema Presto schema (Accumulo namespace)
- */
- public void ensureNamespace(String schema)
- {
- try {
- // If the table schema is not "default" and the namespace does not exist, create it
- if (!schema.equals(DEFAULT) && !connector.namespaceOperations().exists(schema)) {
- connector.namespaceOperations().create(schema);
- }
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to check for existence or create Accumulo namespace", e);
- }
- catch (NamespaceExistsException e) {
- // Suppress race condition between test for existence and creation
- LOG.warn("NamespaceExistsException suppressed when creating " + schema);
- }
- }
-
- public boolean exists(String table)
- {
- return connector.tableOperations().exists(table);
- }
-
- public void createAccumuloTable(String table)
- {
- try {
- connector.tableOperations().create(table);
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to create Accumulo table", e);
- }
- catch (TableExistsException e) {
- throw new PrestoException(ACCUMULO_TABLE_EXISTS, "Accumulo table already exists", e);
- }
- }
-
- public void setLocalityGroups(String tableName, Map> groups)
- {
- if (groups.isEmpty()) {
- return;
- }
-
- try {
- connector.tableOperations().setLocalityGroups(tableName, groups);
- LOG.debug("Set locality groups for %s to %s", tableName, groups);
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set locality groups", e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to set locality groups, table does not exist", e);
- }
- }
-
- public void setIterator(String table, IteratorSetting setting)
- {
- try {
- // Remove any existing iterator settings of the same name, if applicable
- Map> iterators = connector.tableOperations().listIterators(table);
- if (iterators.containsKey(setting.getName())) {
- connector.tableOperations().removeIterator(table, setting.getName(), iterators.get(setting.getName()));
- }
-
- connector.tableOperations().attachIterator(table, setting);
- }
- catch (AccumuloSecurityException | AccumuloException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set iterator on table " + table, e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to set iterator, table does not exist", e);
- }
- }
-
- public void deleteAccumuloTable(String tableName)
- {
- try {
- connector.tableOperations().delete(tableName);
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to delete Accumulo table", e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to delete Accumulo table, does not exist", e);
- }
- }
-
- public void renameAccumuloTable(String oldName, String newName)
- {
- try {
- connector.tableOperations().rename(oldName, newName);
- }
- catch (AccumuloSecurityException | AccumuloException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to rename table", e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to rename table, old table does not exist", e);
- }
- catch (TableExistsException e) {
- throw new PrestoException(ACCUMULO_TABLE_EXISTS, "Failed to rename table, new table already exists", e);
- }
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTransactionHandle.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTransactionHandle.java
deleted file mode 100644
index 317bcdcc0116..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/AccumuloTransactionHandle.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-
-import java.util.Objects;
-import java.util.UUID;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloTransactionHandle
- implements ConnectorTransactionHandle
-{
- private final UUID uuid;
-
- public AccumuloTransactionHandle()
- {
- this(UUID.randomUUID());
- }
-
- @JsonCreator
- public AccumuloTransactionHandle(@JsonProperty("uuid") UUID uuid)
- {
- this.uuid = requireNonNull(uuid, "uuid is null");
- }
-
- @JsonProperty
- public UUID getUuid()
- {
- return uuid;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- return Objects.equals(uuid, ((AccumuloTransactionHandle) obj).uuid);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(uuid);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this).add("uuid", uuid).toString();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/Types.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/Types.java
deleted file mode 100644
index ba5951ebebda..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/Types.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo;
-
-import io.prestosql.spi.type.ArrayType;
-import io.prestosql.spi.type.MapType;
-import io.prestosql.spi.type.Type;
-
-/**
- * Utility class for Presto Type-related functionality.
- */
-public final class Types
-{
- private Types() {}
-
- public static boolean isArrayType(Type type)
- {
- return type instanceof ArrayType;
- }
-
- public static boolean isMapType(Type type)
- {
- return type instanceof MapType;
- }
-
- /**
- * Gets the element type of the given array type. Does not validate that the given type is an array.
- *
- * @param type An array type
- * @return Element type of the array
- * @throws IndexOutOfBoundsException If type is not an array
- * @see Types#isArrayType
- */
- public static Type getElementType(Type type)
- {
- return type.getTypeParameters().get(0);
- }
-
- /**
- * Gets the key type of the given map type. Does not validate that the given type is a map.
- *
- * @param type A map type
- * @return Key type of the map
- * @throws IndexOutOfBoundsException If type is not a map
- * @see Types#isMapType
- */
- public static Type getKeyType(Type type)
- {
- return type.getTypeParameters().get(0);
- }
-
- /**
- * Gets the value type of the given map type. Does not validate that the given type is a map.
- *
- * @param type A map type
- * @return Value type of the map
- * @throws IndexOutOfBoundsException If type is not a map
- * @see Types#isMapType
- */
- public static Type getValueType(Type type)
- {
- return type.getTypeParameters().get(1);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloConfig.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloConfig.java
deleted file mode 100644
index 7939ab88446d..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloConfig.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.conf;
-
-import io.airlift.configuration.Config;
-import io.airlift.configuration.ConfigDescription;
-import io.airlift.configuration.ConfigSecuritySensitive;
-import io.airlift.units.Duration;
-
-import javax.validation.constraints.Min;
-import javax.validation.constraints.NotNull;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * File-based configuration properties for the Accumulo connector
- */
-public class AccumuloConfig
-{
- public static final String INSTANCE = "accumulo.instance";
- public static final String ZOOKEEPERS = "accumulo.zookeepers";
- public static final String USERNAME = "accumulo.username";
- public static final String PASSWORD = "accumulo.password";
- public static final String ZOOKEEPER_METADATA_ROOT = "accumulo.zookeeper.metadata.root";
- public static final String CARDINALITY_CACHE_SIZE = "accumulo.cardinality.cache.size";
- public static final String CARDINALITY_CACHE_EXPIRE_DURATION = "accumulo.cardinality.cache.expire.duration";
-
- private String instance;
- private String zooKeepers;
- private String username;
- private String password;
- private String zkMetadataRoot = "/presto-accumulo";
- private int cardinalityCacheSize = 100_000;
- private Duration cardinalityCacheExpiration = new Duration(5, TimeUnit.MINUTES);
-
- @NotNull
- public String getInstance()
- {
- return this.instance;
- }
-
- @Config(INSTANCE)
- @ConfigDescription("Accumulo instance name")
- public AccumuloConfig setInstance(String instance)
- {
- this.instance = instance;
- return this;
- }
-
- @NotNull
- public String getZooKeepers()
- {
- return this.zooKeepers;
- }
-
- @Config(ZOOKEEPERS)
- @ConfigDescription("ZooKeeper quorum connect string for Accumulo")
- public AccumuloConfig setZooKeepers(String zooKeepers)
- {
- this.zooKeepers = zooKeepers;
- return this;
- }
-
- @NotNull
- public String getUsername()
- {
- return this.username;
- }
-
- @Config(USERNAME)
- @ConfigDescription("Sets the user to use when interacting with Accumulo. This user will require administrative permissions")
- public AccumuloConfig setUsername(String username)
- {
- this.username = username;
- return this;
- }
-
- @NotNull
- public String getPassword()
- {
- return this.password;
- }
-
- @Config(PASSWORD)
- @ConfigSecuritySensitive
- @ConfigDescription("Sets the password for the configured user")
- public AccumuloConfig setPassword(String password)
- {
- this.password = password;
- return this;
- }
-
- @NotNull
- public String getZkMetadataRoot()
- {
- return zkMetadataRoot;
- }
-
- @Config(ZOOKEEPER_METADATA_ROOT)
- @ConfigDescription("Sets the root znode for metadata storage")
- public void setZkMetadataRoot(String zkMetadataRoot)
- {
- this.zkMetadataRoot = zkMetadataRoot;
- }
-
- @NotNull
- @Min(1)
- public int getCardinalityCacheSize()
- {
- return cardinalityCacheSize;
- }
-
- @Config(CARDINALITY_CACHE_SIZE)
- @ConfigDescription("Sets the cardinality cache size")
- public void setCardinalityCacheSize(int cardinalityCacheSize)
- {
- this.cardinalityCacheSize = cardinalityCacheSize;
- }
-
- @NotNull
- public Duration getCardinalityCacheExpiration()
- {
- return cardinalityCacheExpiration;
- }
-
- @Config(CARDINALITY_CACHE_EXPIRE_DURATION)
- @ConfigDescription("Sets the cardinality cache expiration")
- public void setCardinalityCacheExpiration(Duration cardinalityCacheExpiration)
- {
- this.cardinalityCacheExpiration = cardinalityCacheExpiration;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloSessionProperties.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloSessionProperties.java
deleted file mode 100644
index fee1f433e314..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloSessionProperties.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.conf;
-
-import com.google.common.collect.ImmutableList;
-import io.airlift.units.Duration;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.session.PropertyMetadata;
-
-import javax.inject.Inject;
-
-import java.util.List;
-
-import static io.prestosql.spi.session.PropertyMetadata.booleanProperty;
-import static io.prestosql.spi.session.PropertyMetadata.doubleProperty;
-import static io.prestosql.spi.session.PropertyMetadata.integerProperty;
-import static io.prestosql.spi.session.PropertyMetadata.stringProperty;
-import static io.prestosql.spi.type.VarcharType.VARCHAR;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-/**
- * Class contains all session-based properties for the Accumulo connector.
- * Use SHOW SESSION to view all available properties in the Presto CLI.
- *
- * Can set the property using:
- *
- * SET SESSION <property> = <value>;
- */
-public final class AccumuloSessionProperties
-{
- private static final String OPTIMIZE_LOCALITY_ENABLED = "optimize_locality_enabled";
- private static final String OPTIMIZE_SPLIT_RANGES_ENABLED = "optimize_split_ranges_enabled";
- private static final String OPTIMIZE_INDEX_ENABLED = "optimize_index_enabled";
- private static final String INDEX_ROWS_PER_SPLIT = "index_rows_per_split";
- private static final String INDEX_THRESHOLD = "index_threshold";
- private static final String INDEX_LOWEST_CARDINALITY_THRESHOLD = "index_lowest_cardinality_threshold";
- private static final String INDEX_METRICS_ENABLED = "index_metrics_enabled";
- private static final String SCAN_USERNAME = "scan_username";
- private static final String INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH = "index_short_circuit_cardinality_fetch";
- private static final String INDEX_CARDINALITY_CACHE_POLLING_DURATION = "index_cardinality_cache_polling_duration";
-
- private final List> sessionProperties;
-
- @Inject
- public AccumuloSessionProperties()
- {
- sessionProperties = ImmutableList.of(
- booleanProperty(
- OPTIMIZE_LOCALITY_ENABLED,
- "Set to true to enable data locality for non-indexed scans. Default true.", true,
- false),
- booleanProperty(
- OPTIMIZE_SPLIT_RANGES_ENABLED,
- "Set to true to split non-indexed queries by tablet splits. Should generally be true.",
- true, false),
- stringProperty(
- SCAN_USERNAME,
- "User to impersonate when scanning the tables. This property trumps the scan_auths table property. Default is the user in the configuration file.", null, false),
- booleanProperty(
- OPTIMIZE_INDEX_ENABLED,
- "Set to true to enable usage of the secondary index on query. Default true.",
- true,
- false),
- integerProperty(
- INDEX_ROWS_PER_SPLIT,
- "The number of Accumulo row IDs that are packed into a single Presto split. Default 10000",
- 10000,
- false),
- doubleProperty(
- INDEX_THRESHOLD,
- "The ratio between number of rows to be scanned based on the index over the total number of rows. If the ratio is below this threshold, the index will be used. Default .2",
- 0.2,
- false),
- doubleProperty(
- INDEX_LOWEST_CARDINALITY_THRESHOLD,
- "The threshold where the column with the lowest cardinality will be used instead of computing an intersection of ranges in the secondary index. Secondary index must be enabled. Default .01",
- 0.01,
- false),
- booleanProperty(
- INDEX_METRICS_ENABLED,
- "Set to true to enable usage of the metrics table to optimize usage of the index. Default true",
- true,
- false),
- booleanProperty(
- INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH,
- "Short circuit the retrieval of index metrics once any column is less than the lowest cardinality threshold. Default true",
- true,
- false),
- durationProperty(
- INDEX_CARDINALITY_CACHE_POLLING_DURATION,
- "Sets the cardinality cache polling duration for short circuit retrieval of index metrics. Default 10ms",
- new Duration(10, MILLISECONDS),
- false));
- }
-
- public List> getSessionProperties()
- {
- return sessionProperties;
- }
-
- public static boolean isOptimizeLocalityEnabled(ConnectorSession session)
- {
- return session.getProperty(OPTIMIZE_LOCALITY_ENABLED, Boolean.class);
- }
-
- public static boolean isOptimizeSplitRangesEnabled(ConnectorSession session)
- {
- return session.getProperty(OPTIMIZE_SPLIT_RANGES_ENABLED, Boolean.class);
- }
-
- public static boolean isOptimizeIndexEnabled(ConnectorSession session)
- {
- return session.getProperty(OPTIMIZE_INDEX_ENABLED, Boolean.class);
- }
-
- public static double getIndexThreshold(ConnectorSession session)
- {
- return session.getProperty(INDEX_THRESHOLD, Double.class);
- }
-
- public static int getNumIndexRowsPerSplit(ConnectorSession session)
- {
- return session.getProperty(INDEX_ROWS_PER_SPLIT, Integer.class);
- }
-
- public static double getIndexSmallCardThreshold(ConnectorSession session)
- {
- return session.getProperty(INDEX_LOWEST_CARDINALITY_THRESHOLD, Double.class);
- }
-
- public static Duration getIndexCardinalityCachePollingDuration(ConnectorSession session)
- {
- return session.getProperty(INDEX_CARDINALITY_CACHE_POLLING_DURATION, Duration.class);
- }
-
- public static boolean isIndexMetricsEnabled(ConnectorSession session)
- {
- return session.getProperty(INDEX_METRICS_ENABLED, Boolean.class);
- }
-
- public static String getScanUsername(ConnectorSession session)
- {
- return session.getProperty(SCAN_USERNAME, String.class);
- }
-
- public static boolean isIndexShortCircuitEnabled(ConnectorSession session)
- {
- return session.getProperty(INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH, Boolean.class);
- }
-
- private static PropertyMetadata durationProperty(String name, String description, Duration defaultValue, boolean hidden)
- {
- return new PropertyMetadata<>(
- name,
- description,
- VARCHAR,
- Duration.class,
- defaultValue,
- hidden,
- value -> Duration.valueOf((String) value),
- Duration::toString);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloTableProperties.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloTableProperties.java
deleted file mode 100644
index 4cb63890156a..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/conf/AccumuloTableProperties.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.conf;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.plugin.accumulo.serializers.LexicoderRowSerializer;
-import io.prestosql.plugin.accumulo.serializers.StringRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.session.PropertyMetadata;
-import io.prestosql.spi.type.VarcharType;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static com.google.common.base.Preconditions.checkState;
-import static io.prestosql.spi.StandardErrorCode.INVALID_TABLE_PROPERTY;
-import static io.prestosql.spi.session.PropertyMetadata.booleanProperty;
-import static io.prestosql.spi.session.PropertyMetadata.stringProperty;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Class contains all table properties for the Accumulo connector. Used when creating a table:
- *
- * CREATE TABLE foo (a VARCHAR, b INT)
- * WITH (column_mapping = 'b:md:b', external = true);
- */
-public final class AccumuloTableProperties
-{
- public static final String COLUMN_MAPPING = "column_mapping";
- public static final String INDEX_COLUMNS = "index_columns";
- public static final String EXTERNAL = "external";
- public static final String LOCALITY_GROUPS = "locality_groups";
- public static final String ROW_ID = "row_id";
- public static final String SERIALIZER = "serializer";
- public static final String SCAN_AUTHS = "scan_auths";
- private static final Splitter COLON_SPLITTER = Splitter.on(':').trimResults();
- private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults();
- private static final Splitter PIPE_SPLITTER = Splitter.on('|').omitEmptyStrings().trimResults();
-
- private final List> tableProperties;
-
- public AccumuloTableProperties()
- {
- PropertyMetadata s1 = stringProperty(
- COLUMN_MAPPING,
- "Comma-delimited list of column metadata: col_name:col_family:col_qualifier,[...]. Required for external tables. Not setting this property results in auto-generated column names.",
- null,
- false);
-
- PropertyMetadata s2 = stringProperty(
- INDEX_COLUMNS,
- "A comma-delimited list of Presto columns that are indexed in this table's corresponding index table. Default is no indexed columns.",
- "",
- false);
-
- PropertyMetadata s3 = booleanProperty(
- EXTERNAL,
- "If true, Presto will only do metadata operations for the table. Else, Presto will create and drop Accumulo tables where appropriate. Default false.",
- false,
- false);
-
- PropertyMetadata s4 = stringProperty(
- LOCALITY_GROUPS,
- "List of locality groups to set on the Accumulo table. Only valid on internal tables. String format is locality group name, colon, comma delimited list of Presto column names in the group. Groups are delimited by pipes. Example: group1:colA,colB,colC|group2:colD,colE,colF|etc.... Default is no locality groups.",
- null,
- false);
-
- PropertyMetadata s5 = stringProperty(
- ROW_ID,
- "Presto column name that maps to the Accumulo row ID. Default is the first column.",
- null,
- false);
-
- PropertyMetadata s6 = new PropertyMetadata<>(
- SERIALIZER,
- "Serializer for Accumulo data encodings. Can either be 'default', 'string', 'lexicoder', or a Java class name. Default is 'default', i.e. the value from AccumuloRowSerializer.getDefault(), i.e. 'lexicoder'.",
- VarcharType.VARCHAR, String.class,
- AccumuloRowSerializer.getDefault().getClass().getName(),
- false,
- x -> x.equals("default")
- ? AccumuloRowSerializer.getDefault().getClass().getName()
- : (x.equals("string") ? StringRowSerializer.class.getName()
- : (x.equals("lexicoder")
- ? LexicoderRowSerializer.class.getName()
- : (String) x)),
- object -> object);
-
- PropertyMetadata s7 = stringProperty(
- SCAN_AUTHS,
- "Scan-time authorizations set on the batch scanner. Default is all scan authorizations for the user",
- null,
- false);
-
- tableProperties = ImmutableList.of(s1, s2, s3, s4, s5, s6, s7);
- }
-
- public List> getTableProperties()
- {
- return tableProperties;
- }
-
- /**
- * Gets the value of the column_mapping property, or Optional.empty() if not set.
- *
- * Parses the value into a map of Presto column name to a pair of strings, the Accumulo column family and qualifier.
- *
- * @param tableProperties The map of table properties
- * @return The column mapping, presto name to (accumulo column family, qualifier)
- */
- public static Optional>> getColumnMapping(
- Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String strMapping = (String) tableProperties.get(COLUMN_MAPPING);
- if (strMapping == null) {
- return Optional.empty();
- }
-
- // Parse out the column mapping
- // This is a comma-delimited list of "(presto, column:accumulo, fam:accumulo qualifier)" triplets
- ImmutableMap.Builder> mapping = ImmutableMap.builder();
- for (String m : COMMA_SPLITTER.split(strMapping)) {
- String[] tokens = Iterables.toArray(COLON_SPLITTER.split(m), String.class);
- checkState(tokens.length == 3, "Mapping of %s contains %s tokens instead of 3", m, tokens.length);
- mapping.put(tokens[0], Pair.of(tokens[1], tokens[2]));
- }
-
- return Optional.of(mapping.build());
- }
-
- public static Optional> getIndexColumns(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String indexColumns = (String) tableProperties.get(INDEX_COLUMNS);
- if (indexColumns == null) {
- return Optional.empty();
- }
-
- return Optional.of(Arrays.asList(StringUtils.split(indexColumns, ',')));
- }
-
- /**
- * Gets the configured locality groups for the table, or Optional.empty() if not set.
- *
- * All strings are lowercase.
- *
- * @param tableProperties The map of table properties
- * @return Optional map of locality groups
- */
- public static Optional>> getLocalityGroups(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String groupStr = (String) tableProperties.get(LOCALITY_GROUPS);
- if (groupStr == null) {
- return Optional.empty();
- }
-
- ImmutableMap.Builder> groups = ImmutableMap.builder();
-
- // Split all configured locality groups
- for (String group : PIPE_SPLITTER.split(groupStr)) {
- String[] locGroups = Iterables.toArray(COLON_SPLITTER.split(group), String.class);
-
- if (locGroups.length != 2) {
- throw new PrestoException(INVALID_TABLE_PROPERTY, "Locality groups string is malformed. See documentation for proper format.");
- }
-
- String grpName = locGroups[0];
- ImmutableSet.Builder colSet = ImmutableSet.builder();
-
- for (String f : COMMA_SPLITTER.split(locGroups[1])) {
- colSet.add(f.toLowerCase(Locale.ENGLISH));
- }
-
- groups.put(grpName.toLowerCase(Locale.ENGLISH), colSet.build());
- }
-
- return Optional.of(groups.build());
- }
-
- public static Optional getRowId(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String rowId = (String) tableProperties.get(ROW_ID);
- return Optional.ofNullable(rowId);
- }
-
- public static Optional getScanAuthorizations(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String scanAuths = (String) tableProperties.get(SCAN_AUTHS);
- return Optional.ofNullable(scanAuths);
- }
-
- /**
- * Gets the {@link AccumuloRowSerializer} class name to use for this table
- *
- * @param tableProperties The map of table properties
- * @return The name of the AccumuloRowSerializer class
- */
- public static String getSerializerClass(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- String serializerClass = (String) tableProperties.get(SERIALIZER);
- return serializerClass;
- }
-
- public static boolean isExternal(Map tableProperties)
- {
- requireNonNull(tableProperties);
-
- @SuppressWarnings("unchecked")
- Boolean serializerClass = (Boolean) tableProperties.get(EXTERNAL);
- return serializerClass;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/ColumnCardinalityCache.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/ColumnCardinalityCache.java
deleted file mode 100644
index 4160e8fb1b3b..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/ColumnCardinalityCache.java
+++ /dev/null
@@ -1,410 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.index;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.MultimapBuilder;
-import io.airlift.concurrent.BoundedExecutor;
-import io.airlift.log.Logger;
-import io.airlift.units.Duration;
-import io.prestosql.plugin.accumulo.conf.AccumuloConfig;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnConstraint;
-import io.prestosql.spi.PrestoException;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.io.Text;
-
-import javax.annotation.PreDestroy;
-import javax.inject.Inject;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static com.google.common.collect.Streams.stream;
-import static io.airlift.concurrent.Threads.daemonThreadsNamed;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static io.prestosql.plugin.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT;
-import static io.prestosql.plugin.accumulo.index.Indexer.getIndexColumnFamily;
-import static io.prestosql.plugin.accumulo.index.Indexer.getMetricsTableName;
-import static io.prestosql.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR;
-import static java.lang.Long.parseLong;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
-import static java.util.concurrent.Executors.newCachedThreadPool;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-/**
- * This class is an indexing utility to cache the cardinality of a column value for every table.
- * Each table has its own cache that is independent of every other, and every column also has its
- * own Guava cache. Use of this utility can have a significant impact for retrieving the cardinality
- * of many columns, preventing unnecessary accesses to the metrics table in Accumulo for a
- * cardinality that won't change much.
- */
-public class ColumnCardinalityCache
-{
- private static final Logger LOG = Logger.get(ColumnCardinalityCache.class);
- private final Connector connector;
- private final ExecutorService coreExecutor;
- private final BoundedExecutor executorService;
- private final LoadingCache cache;
-
- @Inject
- public ColumnCardinalityCache(Connector connector, AccumuloConfig config)
- {
- this.connector = requireNonNull(connector, "connector is null");
- int size = requireNonNull(config, "config is null").getCardinalityCacheSize();
- Duration expireDuration = config.getCardinalityCacheExpiration();
-
- // Create a bounded executor with a pool size at 4x number of processors
- this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s"));
- this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors());
-
- LOG.debug("Created new cache size %d expiry %s", size, expireDuration);
- cache = CacheBuilder.newBuilder()
- .maximumSize(size)
- .expireAfterWrite(expireDuration.toMillis(), MILLISECONDS)
- .build(new CardinalityCacheLoader());
- }
-
- @PreDestroy
- public void shutdown()
- {
- coreExecutor.shutdownNow();
- }
-
- /**
- * Gets the cardinality for each {@link AccumuloColumnConstraint}.
- * Given constraints are expected to be indexed! Who knows what would happen if they weren't!
- *
- * @param schema Schema name
- * @param table Table name
- * @param auths Scan authorizations
- * @param idxConstraintRangePairs Mapping of all ranges for a given constraint
- * @param earlyReturnThreshold Smallest acceptable cardinality to return early while other tasks complete
- * @param pollingDuration Duration for polling the cardinality completion service
- * @return An immutable multimap of cardinality to column constraint, sorted by cardinality from smallest to largest
- * @throws TableNotFoundException If the metrics table does not exist
- * @throws ExecutionException If another error occurs; I really don't even know anymore.
- */
- public Multimap getCardinalities(String schema, String table, Authorizations auths, Multimap idxConstraintRangePairs, long earlyReturnThreshold, Duration pollingDuration)
- {
- // Submit tasks to the executor to fetch column cardinality, adding it to the Guava cache if necessary
- CompletionService> executor = new ExecutorCompletionService<>(executorService);
- idxConstraintRangePairs.asMap().forEach((key, value) -> executor.submit(() -> {
- long cardinality = getColumnCardinality(schema, table, auths, key.getFamily(), key.getQualifier(), value);
- LOG.debug("Cardinality for column %s is %s", key.getName(), cardinality);
- return Pair.of(cardinality, key);
- }));
-
- // Create a multi map sorted by cardinality
- ListMultimap cardinalityToConstraints = MultimapBuilder.treeKeys().arrayListValues().build();
- try {
- boolean earlyReturn = false;
- int numTasks = idxConstraintRangePairs.asMap().entrySet().size();
- do {
- // Sleep for the polling duration to allow concurrent tasks to run for this time
- Thread.sleep(pollingDuration.toMillis());
-
- // Poll each task, retrieving the result if it is done
- for (int i = 0; i < numTasks; ++i) {
- Future> futureCardinality = executor.poll();
- if (futureCardinality != null && futureCardinality.isDone()) {
- Pair columnCardinality = futureCardinality.get();
- cardinalityToConstraints.put(columnCardinality.getLeft(), columnCardinality.getRight());
- }
- }
-
- // If the smallest cardinality is present and below the threshold, set the earlyReturn flag
- Optional> smallestCardinality = cardinalityToConstraints.entries().stream().findFirst();
- if (smallestCardinality.isPresent()) {
- if (smallestCardinality.get().getKey() <= earlyReturnThreshold) {
- LOG.info("Cardinality %s, is below threshold. Returning early while other tasks finish", smallestCardinality);
- earlyReturn = true;
- }
- }
- }
- while (!earlyReturn && cardinalityToConstraints.entries().size() < numTasks);
- }
- catch (ExecutionException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting cardinality", e);
- }
-
- // Create a copy of the cardinalities
- return ImmutableMultimap.copyOf(cardinalityToConstraints);
- }
-
- /**
- * Gets the column cardinality for all of the given range values. May reach out to the
- * metrics table in Accumulo to retrieve new cache elements.
- *
- * @param schema Table schema
- * @param table Table name
- * @param auths Scan authorizations
- * @param family Accumulo column family
- * @param qualifier Accumulo column qualifier
- * @param colValues All range values to summarize for the cardinality
- * @return The cardinality of the column
- */
- public long getColumnCardinality(String schema, String table, Authorizations auths, String family, String qualifier, Collection colValues)
- throws ExecutionException
- {
- LOG.debug("Getting cardinality for %s:%s", family, qualifier);
-
- // Collect all exact Accumulo Ranges, i.e. single value entries vs. a full scan
- Collection exactRanges = colValues.stream()
- .filter(ColumnCardinalityCache::isExact)
- .map(range -> new CacheKey(schema, table, family, qualifier, range, auths))
- .collect(Collectors.toList());
-
- LOG.debug("Column values contain %s exact ranges of %s", exactRanges.size(), colValues.size());
-
- // Sum the cardinalities for the exact-value Ranges
- // This is where the reach-out to Accumulo occurs for all Ranges that have not
- // previously been fetched
- long sum = cache.getAll(exactRanges).values().stream().mapToLong(Long::longValue).sum();
-
- // If these collection sizes are not equal,
- // then there is at least one non-exact range
- if (exactRanges.size() != colValues.size()) {
- // for each range in the column value
- for (Range range : colValues) {
- // if this range is not exact
- if (!isExact(range)) {
- // Then get the value for this range using the single-value cache lookup
- sum += cache.get(new CacheKey(schema, table, family, qualifier, range, auths));
- }
- }
- }
-
- return sum;
- }
-
- private static boolean isExact(Range range)
- {
- return !range.isInfiniteStartKey() && !range.isInfiniteStopKey() &&
- range.getStartKey().followingKey(PartialKey.ROW).equals(range.getEndKey());
- }
-
- /**
- * Complex key for the CacheLoader
- */
- private static class CacheKey
- {
- private final String schema;
- private final String table;
- private final String family;
- private final String qualifier;
- private final Range range;
- private final Authorizations auths;
-
- public CacheKey(
- String schema,
- String table,
- String family,
- String qualifier,
- Range range,
- Authorizations auths)
- {
- this.schema = requireNonNull(schema, "schema is null");
- this.table = requireNonNull(table, "table is null");
- this.family = requireNonNull(family, "family is null");
- this.qualifier = requireNonNull(qualifier, "qualifier is null");
- this.range = requireNonNull(range, "range is null");
- this.auths = requireNonNull(auths, "auths is null");
- }
-
- public String getSchema()
- {
- return schema;
- }
-
- public String getTable()
- {
- return table;
- }
-
- public String getFamily()
- {
- return family;
- }
-
- public String getQualifier()
- {
- return qualifier;
- }
-
- public Range getRange()
- {
- return range;
- }
-
- public Authorizations getAuths()
- {
- return auths;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(schema, table, family, qualifier, range);
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
-
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- CacheKey other = (CacheKey) obj;
- return Objects.equals(this.schema, other.schema)
- && Objects.equals(this.table, other.table)
- && Objects.equals(this.family, other.family)
- && Objects.equals(this.qualifier, other.qualifier)
- && Objects.equals(this.range, other.range);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("schema", schema)
- .add("table", table)
- .add("family", family)
- .add("qualifier", qualifier)
- .add("range", range).toString();
- }
- }
-
- /**
- * Internal class for loading the cardinality from Accumulo
- */
- private class CardinalityCacheLoader
- extends CacheLoader
- {
- /**
- * Loads the cardinality for the given Range. Uses a BatchScanner and sums the cardinality for all values that encapsulate the Range.
- *
- * @param key Range to get the cardinality for
- * @return The cardinality of the column, which would be zero if the value does not exist
- */
- @Override
- public Long load(CacheKey key)
- throws Exception
- {
- LOG.debug("Loading a non-exact range from Accumulo: %s", key);
- // Get metrics table name and the column family for the scanner
- String metricsTable = getMetricsTableName(key.getSchema(), key.getTable());
- Text columnFamily = new Text(getIndexColumnFamily(key.getFamily().getBytes(UTF_8), key.getQualifier().getBytes(UTF_8)).array());
-
- // Create scanner for querying the range
- BatchScanner scanner = connector.createBatchScanner(metricsTable, key.auths, 10);
- scanner.setRanges(connector.tableOperations().splitRangeByTablets(metricsTable, key.range, Integer.MAX_VALUE));
- scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT);
-
- try {
- return stream(scanner)
- .map(Entry::getValue)
- .map(Value::toString)
- .mapToLong(Long::parseLong)
- .sum();
- }
- finally {
- scanner.close();
- }
- }
-
- @Override
- public Map loadAll(Iterable keys)
- throws Exception
- {
- int size = Iterables.size(keys);
- if (size == 0) {
- return ImmutableMap.of();
- }
-
- LOG.debug("Loading %s exact ranges from Accumulo", size);
-
- // In order to simplify the implementation, we are making a (safe) assumption
- // that the CacheKeys will all contain the same combination of schema/table/family/qualifier
- // This is asserted with the below implementation error just to make sure
- CacheKey anyKey = stream(keys).findAny().get();
- if (stream(keys).anyMatch(k -> !k.getSchema().equals(anyKey.getSchema()) || !k.getTable().equals(anyKey.getTable()) || !k.getFamily().equals(anyKey.getFamily()) || !k.getQualifier().equals(anyKey.getQualifier()))) {
- throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "loadAll called with a non-homogeneous collection of cache keys");
- }
-
- Map rangeToKey = stream(keys).collect(Collectors.toMap(CacheKey::getRange, Function.identity()));
- LOG.debug("rangeToKey size is %s", rangeToKey.size());
-
- // Get metrics table name and the column family for the scanner
- String metricsTable = getMetricsTableName(anyKey.getSchema(), anyKey.getTable());
- Text columnFamily = new Text(getIndexColumnFamily(anyKey.getFamily().getBytes(UTF_8), anyKey.getQualifier().getBytes(UTF_8)).array());
-
- BatchScanner scanner = connector.createBatchScanner(metricsTable, anyKey.getAuths(), 10);
- try {
- scanner.setRanges(stream(keys).map(CacheKey::getRange).collect(Collectors.toList()));
- scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT);
-
- // Create a new map to hold our cardinalities for each range, returning a default of
- // Zero for each non-existent Key
- Map rangeValues = new HashMap<>();
- stream(keys).forEach(key -> rangeValues.put(key, 0L));
-
- for (Entry entry : scanner) {
- rangeValues.put(rangeToKey.get(Range.exact(entry.getKey().getRow())), parseLong(entry.getValue().toString()));
- }
-
- return rangeValues;
- }
- finally {
- if (scanner != null) {
- scanner.close();
- }
- }
- }
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/IndexLookup.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/IndexLookup.java
deleted file mode 100644
index 759f9b174a11..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/IndexLookup.java
+++ /dev/null
@@ -1,401 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.index;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Multimap;
-import io.airlift.concurrent.BoundedExecutor;
-import io.airlift.log.Logger;
-import io.airlift.units.Duration;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnConstraint;
-import io.prestosql.plugin.accumulo.model.TabletSplitMetadata;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ConnectorSession;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.hadoop.io.Text;
-
-import javax.annotation.PreDestroy;
-import javax.inject.Inject;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static io.airlift.concurrent.Threads.daemonThreadsNamed;
-import static io.prestosql.plugin.accumulo.AccumuloClient.getRangesFromDomain;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.getIndexCardinalityCachePollingDuration;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.getIndexSmallCardThreshold;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.getIndexThreshold;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.getNumIndexRowsPerSplit;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.isIndexMetricsEnabled;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.isIndexShortCircuitEnabled;
-import static io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties.isOptimizeIndexEnabled;
-import static io.prestosql.plugin.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT;
-import static io.prestosql.plugin.accumulo.index.Indexer.METRICS_TABLE_ROWID_AS_TEXT;
-import static io.prestosql.plugin.accumulo.index.Indexer.METRICS_TABLE_ROWS_CF_AS_TEXT;
-import static io.prestosql.plugin.accumulo.index.Indexer.getIndexTableName;
-import static io.prestosql.plugin.accumulo.index.Indexer.getMetricsTableName;
-import static io.prestosql.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
-import static java.util.concurrent.Executors.newCachedThreadPool;
-
-/**
- * Class to assist the Presto connector, and maybe external applications,
- * leverage the secondary * index built by the {@link Indexer}.
- * Leverages {@link ColumnCardinalityCache} to assist in * retrieving row IDs.
- * Currently pretty bound to the Presto connector APIs.
- */
-public class IndexLookup
-{
- private static final Logger LOG = Logger.get(IndexLookup.class);
- private static final Range METRICS_TABLE_ROWID_RANGE = new Range(METRICS_TABLE_ROWID_AS_TEXT);
- private final ColumnCardinalityCache cardinalityCache;
- private final Connector connector;
- private final ExecutorService coreExecutor;
- private final BoundedExecutor executorService;
-
- @Inject
- public IndexLookup(Connector connector, ColumnCardinalityCache cardinalityCache)
- {
- this.connector = requireNonNull(connector, "connector is null");
- this.cardinalityCache = requireNonNull(cardinalityCache, "cardinalityCache is null");
-
- // Create a bounded executor with a pool size at 4x number of processors
- this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s"));
- this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors());
- }
-
- @PreDestroy
- public void shutdown()
- {
- coreExecutor.shutdownNow();
- }
-
- /**
- * Scans the index table, applying the index based on the given column constraints to return a set of tablet splits.
- *
- * If this function returns true, the output parameter tabletSplits contains a list of TabletSplitMetadata objects.
- * These in turn contain a collection of Ranges containing the exact row IDs determined using the index.
- *
- * If this function returns false, the secondary index should not be used. In this case,
- * either the accumulo session has disabled secondary indexing,
- * or the number of row IDs that would be used by the secondary index is greater than the configured threshold
- * (again retrieved from the session).
- *
- * @param schema Schema name
- * @param table Table name
- * @param session Current client session
- * @param constraints All column constraints (this method will filter for if the column is indexed)
- * @param rowIdRanges Collection of Accumulo ranges based on any predicate against a record key
- * @param tabletSplits Output parameter containing the bundles of row IDs determined by the use of the index.
- * @param serializer Instance of a row serializer
- * @param auths Scan-time authorizations
- * @return True if the tablet splits are valid and should be used, false otherwise
- * @throws Exception If something bad happens. What are the odds?
- */
- public boolean applyIndex(
- String schema,
- String table,
- ConnectorSession session,
- Collection constraints,
- Collection rowIdRanges,
- List tabletSplits,
- AccumuloRowSerializer serializer,
- Authorizations auths)
- throws Exception
- {
- // Early out if index is disabled
- if (!isOptimizeIndexEnabled(session)) {
- LOG.debug("Secondary index is disabled");
- return false;
- }
-
- LOG.debug("Secondary index is enabled");
-
- // Collect Accumulo ranges for each indexed column constraint
- Multimap constraintRanges = getIndexedConstraintRanges(constraints, serializer);
-
- // If there is no constraints on an index column, we again will bail out
- if (constraintRanges.isEmpty()) {
- LOG.debug("Query contains no constraints on indexed columns, skipping secondary index");
- return false;
- }
-
- // If metrics are not enabled
- if (!isIndexMetricsEnabled(session)) {
- LOG.debug("Use of index metrics is disabled");
- // Get the ranges via the index table
- List indexRanges = getIndexRanges(getIndexTableName(schema, table), constraintRanges, rowIdRanges, auths);
-
- if (!indexRanges.isEmpty()) {
- // Bin the ranges into TabletMetadataSplits and return true to use the tablet splits
- binRanges(getNumIndexRowsPerSplit(session), indexRanges, tabletSplits);
- LOG.debug("Number of splits for %s.%s is %d with %d ranges", schema, table, tabletSplits.size(), indexRanges.size());
- }
- else {
- LOG.debug("Query would return no results, returning empty list of splits");
- }
-
- return true;
- }
- else {
- LOG.debug("Use of index metrics is enabled");
- // Get ranges using the metrics
- return getRangesWithMetrics(session, schema, table, constraintRanges, rowIdRanges, tabletSplits, auths);
- }
- }
-
- private static Multimap getIndexedConstraintRanges(Collection constraints, AccumuloRowSerializer serializer)
- {
- ImmutableListMultimap.Builder builder = ImmutableListMultimap.builder();
- for (AccumuloColumnConstraint columnConstraint : constraints) {
- if (columnConstraint.isIndexed()) {
- for (Range range : getRangesFromDomain(columnConstraint.getDomain(), serializer)) {
- builder.put(columnConstraint, range);
- }
- }
- else {
- LOG.warn("Query contains constraint on non-indexed column %s. Is it worth indexing?", columnConstraint.getName());
- }
- }
- return builder.build();
- }
-
- private boolean getRangesWithMetrics(
- ConnectorSession session,
- String schema,
- String table,
- Multimap constraintRanges,
- Collection rowIdRanges,
- List tabletSplits,
- Authorizations auths)
- throws Exception
- {
- String metricsTable = getMetricsTableName(schema, table);
- long numRows = getNumRowsInTable(metricsTable, auths);
-
- // Get the cardinalities from the metrics table
- Multimap cardinalities;
- if (isIndexShortCircuitEnabled(session)) {
- cardinalities = cardinalityCache.getCardinalities(
- schema,
- table,
- auths,
- constraintRanges,
- (long) (numRows * getIndexSmallCardThreshold(session)),
- getIndexCardinalityCachePollingDuration(session));
- }
- else {
- // disable short circuit using 0
- cardinalities = cardinalityCache.getCardinalities(schema, table, auths, constraintRanges, 0, new Duration(0, TimeUnit.MILLISECONDS));
- }
-
- Optional> entry = cardinalities.entries().stream().findFirst();
- if (!entry.isPresent()) {
- return false;
- }
-
- Entry lowestCardinality = entry.get();
- String indexTable = getIndexTableName(schema, table);
- double threshold = getIndexThreshold(session);
- List indexRanges;
-
- // If the smallest cardinality in our list is above the lowest cardinality threshold,
- // we should look at intersecting the row ID ranges to try and get under the threshold.
- if (smallestCardAboveThreshold(session, numRows, lowestCardinality.getKey())) {
- // If we only have one column, we can skip the intersection process and just check the index threshold
- if (cardinalities.size() == 1) {
- long numEntries = lowestCardinality.getKey();
- double ratio = ((double) numEntries / (double) numRows);
- LOG.debug("Use of index would scan %s of %s rows, ratio %s. Threshold %2f, Using for index table? %s", numEntries, numRows, ratio, threshold, ratio < threshold);
- if (ratio >= threshold) {
- return false;
- }
- }
-
- // Else, get the intersection of all row IDs for all column constraints
- LOG.debug("%d indexed columns, intersecting ranges", constraintRanges.size());
- indexRanges = getIndexRanges(indexTable, constraintRanges, rowIdRanges, auths);
- LOG.debug("Intersection results in %d ranges from secondary index", indexRanges.size());
- }
- else {
- // Else, we don't need to intersect the columns and we can just use the column with the lowest cardinality,
- // so get all those row IDs in a set of ranges.
- LOG.debug("Not intersecting columns, using column with lowest cardinality ");
- ImmutableMultimap.Builder lcBldr = ImmutableMultimap.builder();
- lcBldr.putAll(lowestCardinality.getValue(), constraintRanges.get(lowestCardinality.getValue()));
- indexRanges = getIndexRanges(indexTable, lcBldr.build(), rowIdRanges, auths);
- }
-
- if (indexRanges.isEmpty()) {
- LOG.debug("Query would return no results, returning empty list of splits");
- return true;
- }
-
- // Okay, we now check how many rows we would scan by using the index vs. the overall number
- // of rows
- long numEntries = indexRanges.size();
- double ratio = (double) numEntries / (double) numRows;
- LOG.debug("Use of index would scan %d of %d rows, ratio %s. Threshold %2f, Using for table? %b", numEntries, numRows, ratio, threshold, ratio < threshold, table);
-
- // If the percentage of scanned rows, the ratio, less than the configured threshold
- if (ratio < threshold) {
- // Bin the ranges into TabletMetadataSplits and return true to use the tablet splits
- binRanges(getNumIndexRowsPerSplit(session), indexRanges, tabletSplits);
- LOG.debug("Number of splits for %s.%s is %d with %d ranges", schema, table, tabletSplits.size(), indexRanges.size());
- return true;
- }
- else {
- // We are going to do too much work to use the secondary index, so return false
- return false;
- }
- }
-
- private static boolean smallestCardAboveThreshold(ConnectorSession session, long numRows, long smallestCardinality)
- {
- double ratio = ((double) smallestCardinality / (double) numRows);
- double threshold = getIndexSmallCardThreshold(session);
- LOG.debug("Smallest cardinality is %d, num rows is %d, ratio is %2f with threshold of %f", smallestCardinality, numRows, ratio, threshold);
- return ratio > threshold;
- }
-
- private long getNumRowsInTable(String metricsTable, Authorizations auths)
- throws TableNotFoundException
- {
- // Create scanner against the metrics table, pulling the special column and the rows column
- Scanner scanner = connector.createScanner(metricsTable, auths);
- scanner.setRange(METRICS_TABLE_ROWID_RANGE);
- scanner.fetchColumn(METRICS_TABLE_ROWS_CF_AS_TEXT, CARDINALITY_CQ_AS_TEXT);
-
- // Scan the entry and get the number of rows
- long numRows = -1;
- for (Entry entry : scanner) {
- if (numRows > 0) {
- throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Should have received only one entry when scanning for number of rows in metrics table");
- }
- numRows = Long.parseLong(entry.getValue().toString());
- }
- scanner.close();
-
- LOG.debug("Number of rows in table is %d", numRows);
- return numRows;
- }
-
- private List getIndexRanges(String indexTable, Multimap constraintRanges, Collection rowIDRanges, Authorizations auths)
- {
- Set finalRanges = new HashSet<>();
- // For each column/constraint pair we submit a task to scan the index ranges
- List>> tasks = new ArrayList<>();
- CompletionService> executor = new ExecutorCompletionService<>(executorService);
- for (Entry> constraintEntry : constraintRanges.asMap().entrySet()) {
- tasks.add(executor.submit(() -> {
- // Create a batch scanner against the index table, setting the ranges
- BatchScanner scan = connector.createBatchScanner(indexTable, auths, 10);
- scan.setRanges(constraintEntry.getValue());
-
- // Fetch the column family for this specific column
- scan.fetchColumnFamily(new Text(Indexer.getIndexColumnFamily(constraintEntry.getKey().getFamily().getBytes(UTF_8), constraintEntry.getKey().getQualifier().getBytes(UTF_8)).array()));
-
- // For each entry in the scanner
- Text tmpQualifier = new Text();
- Set columnRanges = new HashSet<>();
- for (Entry entry : scan) {
- entry.getKey().getColumnQualifier(tmpQualifier);
-
- // Add to our column ranges if it is in one of the row ID ranges
- if (inRange(tmpQualifier, rowIDRanges)) {
- columnRanges.add(new Range(tmpQualifier));
- }
- }
-
- LOG.debug("Retrieved %d ranges for index column %s", columnRanges.size(), constraintEntry.getKey().getName());
- scan.close();
- return columnRanges;
- }));
- }
- tasks.forEach(future -> {
- try {
- // If finalRanges is null, we have not yet added any column ranges
- if (finalRanges.isEmpty()) {
- finalRanges.addAll(future.get());
- }
- else {
- // Retain only the row IDs for this column that have already been added
- // This is your set intersection operation!
- finalRanges.retainAll(future.get());
- }
- }
- catch (ExecutionException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting index ranges", e.getCause());
- }
- });
- return ImmutableList.copyOf(finalRanges);
- }
-
- private static void binRanges(int numRangesPerBin, List splitRanges, List prestoSplits)
- {
- checkArgument(numRangesPerBin > 0, "number of ranges per bin must positivebe greater than zero");
- int toAdd = splitRanges.size();
- int fromIndex = 0;
- int toIndex = Math.min(toAdd, numRangesPerBin);
- do {
- // Add the sublist of range handles
- // Use an empty location because we are binning multiple Ranges spread across many tablet servers
- prestoSplits.add(new TabletSplitMetadata(Optional.empty(), splitRanges.subList(fromIndex, toIndex)));
- toAdd -= toIndex - fromIndex;
- fromIndex = toIndex;
- toIndex += Math.min(toAdd, numRangesPerBin);
- }
- while (toAdd > 0);
- }
-
- /**
- * Gets a Boolean value indicating if the given value is in one of the Ranges in the given collection
- *
- * @param text Text object to check against the Range collection
- * @param ranges Ranges to look into
- * @return True if the text object is in one of the ranges, false otherwise
- */
- private static boolean inRange(Text text, Collection ranges)
- {
- Key kCq = new Key(text);
- return ranges.stream().anyMatch(r -> !r.beforeStartKey(kCq) && !r.afterEndKey(kCq));
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/Indexer.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/Indexer.java
deleted file mode 100644
index c9a2ce90e598..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/index/Indexer.java
+++ /dev/null
@@ -1,559 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.index;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.primitives.UnsignedBytes;
-import io.prestosql.plugin.accumulo.Types;
-import io.prestosql.plugin.accumulo.iterators.MaxByteArrayCombiner;
-import io.prestosql.plugin.accumulo.iterators.MinByteArrayCombiner;
-import io.prestosql.plugin.accumulo.metadata.AccumuloTable;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.type.Type;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.ColumnUpdate;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.LongCombiner;
-import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.io.Text;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
-import static java.nio.ByteBuffer.wrap;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
-
-/**
- * This utility class assists the Presto connector, and external applications,
- * in populating the index table and metrics table for Accumulo-backed Presto tables.
- *
- * This class is totally not thread safe.
- *
- * When creating a table, if it contains indexed columns, users will have to create the index table
- * and the index metrics table, the names of which can be retrieved using the static functions in
- * this class. Additionally, users MUST add iterators to the index metrics table (also available via
- * static function), and, while not required, recommended to add the locality groups to the index
- * table to improve index lookup times.
- *
- * Sample usage of an indexer:
- *
- *
- *
- * Indexer indexer = new Indexer(connector, userAuths, table, writerConf);
- * for (Mutation m : mutationsToNormalTable) {
- * indexer.index(m);
- * }
- *
- * // can flush indexer w/regular BatchWriter
- * indexer.flush()
- *
- * // finished adding new mutations, close the indexer
- * indexer.close();
- *
- *
- */
-@NotThreadSafe
-public class Indexer
- implements Closeable
-{
- public static final ByteBuffer METRICS_TABLE_ROW_ID = wrap("___METRICS_TABLE___".getBytes(UTF_8));
- public static final ByteBuffer METRICS_TABLE_ROWS_CF = wrap("___rows___".getBytes(UTF_8));
- public static final MetricsKey METRICS_TABLE_ROW_COUNT = new MetricsKey(METRICS_TABLE_ROW_ID, METRICS_TABLE_ROWS_CF);
- public static final ByteBuffer METRICS_TABLE_FIRST_ROW_CQ = wrap("___first_row___".getBytes(UTF_8));
- public static final ByteBuffer METRICS_TABLE_LAST_ROW_CQ = wrap("___last_row___".getBytes(UTF_8));
- public static final byte[] CARDINALITY_CQ = "___card___".getBytes(UTF_8);
- public static final Text CARDINALITY_CQ_AS_TEXT = new Text(CARDINALITY_CQ);
- public static final Text METRICS_TABLE_ROWS_CF_AS_TEXT = new Text(METRICS_TABLE_ROWS_CF.array());
- public static final Text METRICS_TABLE_ROWID_AS_TEXT = new Text(METRICS_TABLE_ROW_ID.array());
-
- private static final byte[] EMPTY_BYTES = new byte[0];
- private static final byte UNDERSCORE = '_';
- private static final TypedValueCombiner.Encoder ENCODER = new LongCombiner.StringEncoder();
-
- private final AccumuloTable table;
- private final BatchWriter indexWriter;
- private final BatchWriterConfig writerConfig;
- private final Connector connector;
- private final Map metrics = new HashMap<>();
- private final Multimap indexColumns;
- private final Map> indexColumnTypes;
- private final AccumuloRowSerializer serializer;
- private final Comparator byteArrayComparator = UnsignedBytes.lexicographicalComparator();
-
- private byte[] firstRow;
- private byte[] lastRow;
-
- public Indexer(
- Connector connector,
- Authorizations auths,
- AccumuloTable table,
- BatchWriterConfig writerConfig)
- throws TableNotFoundException
- {
- this.connector = requireNonNull(connector, "connector is null");
- this.table = requireNonNull(table, "table is null");
- this.writerConfig = requireNonNull(writerConfig, "writerConfig is null");
- requireNonNull(auths, "auths is null");
-
- this.serializer = table.getSerializerInstance();
-
- // Create our batch writer
- indexWriter = connector.createBatchWriter(table.getIndexTableName(), writerConfig);
-
- ImmutableMultimap.Builder indexColumnsBuilder = ImmutableMultimap.builder();
- Map> indexColumnTypesBuilder = new HashMap<>();
-
- // Initialize metadata
- table.getColumns().forEach(columnHandle -> {
- if (columnHandle.isIndexed()) {
- // Wrap the column family and qualifier for this column and add it to
- // collection of indexed columns
- ByteBuffer family = wrap(columnHandle.getFamily().get().getBytes(UTF_8));
- ByteBuffer qualifier = wrap(columnHandle.getQualifier().get().getBytes(UTF_8));
- indexColumnsBuilder.put(family, qualifier);
-
- // Create a mapping for this column's Presto type, again creating a new one for the
- // family if necessary
- Map types = indexColumnTypesBuilder.get(family);
- if (types == null) {
- types = new HashMap<>();
- indexColumnTypesBuilder.put(family, types);
- }
- types.put(qualifier, columnHandle.getType());
- }
- });
-
- indexColumns = indexColumnsBuilder.build();
- indexColumnTypes = ImmutableMap.copyOf(indexColumnTypesBuilder);
-
- // If there are no indexed columns, throw an exception
- if (indexColumns.isEmpty()) {
- throw new PrestoException(NOT_SUPPORTED, "No indexed columns in table metadata. Refusing to index a table with no indexed columns");
- }
-
- // Initialize metrics map
- // This metrics map is for column cardinality
- metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0));
-
- // Scan the metrics table for existing first row and last row
- Pair minmax = getMinMaxRowIds(connector, table, auths);
- firstRow = minmax.getLeft();
- lastRow = minmax.getRight();
- }
-
- /**
- * Index the given mutation, adding mutations to the index and metrics table
- *
- * Like typical use of a BatchWriter, this method does not flush mutations to the underlying index table.
- * For higher throughput the modifications to the metrics table are tracked in memory and added to the metrics table when the indexer is flushed or closed.
- *
- * @param mutation Mutation to index
- */
- public void index(Mutation mutation)
- {
- // Increment the cardinality for the number of rows in the table
- metrics.get(METRICS_TABLE_ROW_COUNT).incrementAndGet();
-
- // Set the first and last row values of the table based on existing row IDs
- if (firstRow == null || byteArrayComparator.compare(mutation.getRow(), firstRow) < 0) {
- firstRow = mutation.getRow();
- }
-
- if (lastRow == null || byteArrayComparator.compare(mutation.getRow(), lastRow) > 0) {
- lastRow = mutation.getRow();
- }
-
- // For each column update in this mutation
- for (ColumnUpdate columnUpdate : mutation.getUpdates()) {
- // Get the column qualifiers we want to index for this column family (if any)
- ByteBuffer family = wrap(columnUpdate.getColumnFamily());
- Collection indexQualifiers = indexColumns.get(family);
-
- // If we have column qualifiers we want to index for this column family
- if (indexQualifiers != null) {
- // Check if we want to index this particular qualifier
- ByteBuffer qualifier = wrap(columnUpdate.getColumnQualifier());
- if (indexQualifiers.contains(qualifier)) {
- // If so, create a mutation using the following mapping:
- // Row ID = column value
- // Column Family = columnqualifier_columnfamily
- // Column Qualifier = row ID
- // Value = empty
- ByteBuffer indexFamily = getIndexColumnFamily(columnUpdate.getColumnFamily(), columnUpdate.getColumnQualifier());
- Type type = indexColumnTypes.get(family).get(qualifier);
- ColumnVisibility visibility = new ColumnVisibility(columnUpdate.getColumnVisibility());
-
- // If this is an array type, then index each individual element in the array
- if (Types.isArrayType(type)) {
- Type elementType = Types.getElementType(type);
- List elements = serializer.decode(type, columnUpdate.getValue());
- for (Object element : elements) {
- addIndexMutation(wrap(serializer.encode(elementType, element)), indexFamily, visibility, mutation.getRow());
- }
- }
- else {
- addIndexMutation(wrap(columnUpdate.getValue()), indexFamily, visibility, mutation.getRow());
- }
- }
- }
- }
- }
-
- public void index(Iterable mutations)
- {
- for (Mutation mutation : mutations) {
- index(mutation);
- }
- }
-
- private void addIndexMutation(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility, byte[] qualifier)
- {
- // Create the mutation and add it to the batch writer
- Mutation indexMutation = new Mutation(row.array());
- indexMutation.put(family.array(), qualifier, visibility, EMPTY_BYTES);
- try {
- indexWriter.addMutation(indexMutation);
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation rejected by server", e);
- }
-
- // Increment the cardinality metrics for this value of index
- // metrics is a mapping of row ID to column family
- MetricsKey key = new MetricsKey(row, family, visibility);
- AtomicLong count = metrics.get(key);
- if (count == null) {
- count = new AtomicLong(0);
- metrics.put(key, count);
- }
-
- count.incrementAndGet();
- }
-
- /**
- * Flushes all Mutations in the index writer. And all metric mutations to the metrics table.
- * Note that the metrics table is not updated until this method is explicitly called (or implicitly via close).
- */
- public void flush()
- {
- try {
- // Flush index writer
- indexWriter.flush();
-
- // Write out metrics mutations
- BatchWriter metricsWriter = connector.createBatchWriter(table.getMetricsTableName(), writerConfig);
- metricsWriter.addMutations(getMetricsMutations());
- metricsWriter.close();
-
- // Re-initialize the metrics
- metrics.clear();
- metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0));
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation was rejected by server on flush", e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Accumulo table does not exist", e);
- }
- }
-
- /**
- * Flushes all remaining mutations via {@link Indexer#flush} and closes the index writer.
- */
- @Override
- public void close()
- {
- try {
- flush();
- indexWriter.close();
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation was rejected by server on close", e);
- }
- }
-
- private Collection getMetricsMutations()
- {
- ImmutableList.Builder mutationBuilder = ImmutableList.builder();
- // Mapping of column value to column to number of row IDs that contain that value
- for (Entry entry : metrics.entrySet()) {
- // Row ID: Column value
- // Family: columnfamily_columnqualifier
- // Qualifier: CARDINALITY_CQ
- // Visibility: Inherited from indexed Mutation
- // Value: Cardinality
- Mutation mut = new Mutation(entry.getKey().row.array());
- mut.put(entry.getKey().family.array(), CARDINALITY_CQ, entry.getKey().visibility, ENCODER.encode(entry.getValue().get()));
-
- // Add to our list of mutations
- mutationBuilder.add(mut);
- }
-
- // If the first row and last row are both not null,
- // which would really be for a brand new table that has zero rows and no indexed elements...
- // Talk about your edge cases!
- if (firstRow != null && lastRow != null) {
- // Add a some columns to the special metrics table row ID for the first/last row.
- // Note that if the values on the server side are greater/lesser,
- // the configured iterator will take care of this at scan/compaction time
- Mutation firstLastMutation = new Mutation(METRICS_TABLE_ROW_ID.array());
- firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_FIRST_ROW_CQ.array(), firstRow);
- firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_LAST_ROW_CQ.array(), lastRow);
- mutationBuilder.add(firstLastMutation);
- }
-
- return mutationBuilder.build();
- }
-
- /**
- * Gets a collection of iterator settings that should be added to the metric table for the given Accumulo table. Don't forget! Please!
- *
- * @param table Table for retrieving metrics iterators, see AccumuloClient#getTable
- * @return Collection of iterator settings
- */
- public static Collection getMetricIterators(AccumuloTable table)
- {
- String cardQualifier = new String(CARDINALITY_CQ, UTF_8);
- String rowsFamily = new String(METRICS_TABLE_ROWS_CF.array(), UTF_8);
-
- // Build a string for all columns where the summing combiner should be applied,
- // i.e. all indexed columns
- StringBuilder cardBuilder = new StringBuilder(rowsFamily + ":" + cardQualifier + ",");
- for (String s : getLocalityGroups(table).keySet()) {
- cardBuilder.append(s).append(":").append(cardQualifier).append(',');
- }
- cardBuilder.deleteCharAt(cardBuilder.length() - 1);
-
- // Configuration rows for the Min/Max combiners
- String firstRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_FIRST_ROW_CQ.array(), UTF_8);
- String lastRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_LAST_ROW_CQ.array(), UTF_8);
-
- // Summing combiner for cardinality columns
- IteratorSetting s1 = new IteratorSetting(1, SummingCombiner.class, ImmutableMap.of("columns", cardBuilder.toString(), "type", "STRING"));
-
- // Min/Max combiner for the first/last rows of the table
- IteratorSetting s2 = new IteratorSetting(2, MinByteArrayCombiner.class, ImmutableMap.of("columns", firstRowColumn));
- IteratorSetting s3 = new IteratorSetting(3, MaxByteArrayCombiner.class, ImmutableMap.of("columns", lastRowColumn));
-
- return ImmutableList.of(s1, s2, s3);
- }
-
- /**
- * Gets the column family of the index table based on the given column family and qualifier.
- *
- * @param columnFamily Presto column family
- * @param columnQualifier Presto column qualifier
- * @return ByteBuffer of the given index column family
- */
- public static ByteBuffer getIndexColumnFamily(byte[] columnFamily, byte[] columnQualifier)
- {
- return wrap(ArrayUtils.addAll(ArrayUtils.add(columnFamily, UNDERSCORE), columnQualifier));
- }
-
- /**
- * Gets a set of locality groups that should be added to the index table (not the metrics table).
- *
- * @param table Table for the locality groups, see AccumuloClient#getTable
- * @return Mapping of locality group to column families in the locality group, 1:1 mapping in
- * this case
- */
- public static Map> getLocalityGroups(AccumuloTable table)
- {
- Map> groups = new HashMap<>();
- // For each indexed column
- for (AccumuloColumnHandle columnHandle : table.getColumns().stream().filter(AccumuloColumnHandle::isIndexed).collect(Collectors.toList())) {
- // Create a Text version of the index column family
- Text indexColumnFamily = new Text(getIndexColumnFamily(columnHandle.getFamily().get().getBytes(UTF_8), columnHandle.getQualifier().get().getBytes(UTF_8)).array());
-
- // Add this to the locality groups,
- // it is a 1:1 mapping of locality group to column families
- groups.put(indexColumnFamily.toString(), ImmutableSet.of(indexColumnFamily));
- }
- return groups;
- }
-
- /**
- * Gets the fully-qualified index table name for the given table.
- *
- * @param schema Schema name
- * @param table Table name
- * @return Qualified index table name
- */
- public static String getIndexTableName(String schema, String table)
- {
- return schema.equals("default") ? table + "_idx" : schema + '.' + table + "_idx";
- }
-
- /**
- * Gets the fully-qualified index table name for the given table.
- *
- * @param tableName Schema table name
- * @return Qualified index table name
- */
- public static String getIndexTableName(SchemaTableName tableName)
- {
- return getIndexTableName(tableName.getSchemaName(), tableName.getTableName());
- }
-
- /**
- * Gets the fully-qualified index metrics table name for the given table.
- *
- * @param schema Schema name
- * @param table Table name
- * @return Qualified index metrics table name
- */
- public static String getMetricsTableName(String schema, String table)
- {
- return schema.equals("default") ? table + "_idx_metrics"
- : schema + '.' + table + "_idx_metrics";
- }
-
- /**
- * Gets the fully-qualified index metrics table name for the given table.
- *
- * @param tableName Schema table name
- * @return Qualified index metrics table name
- */
- public static String getMetricsTableName(SchemaTableName tableName)
- {
- return getMetricsTableName(tableName.getSchemaName(), tableName.getTableName());
- }
-
- public static Pair getMinMaxRowIds(Connector connector, AccumuloTable table, Authorizations auths)
- throws TableNotFoundException
- {
- Scanner scanner = connector.createScanner(table.getMetricsTableName(), auths);
- scanner.setRange(new Range(new Text(Indexer.METRICS_TABLE_ROW_ID.array())));
- Text family = new Text(Indexer.METRICS_TABLE_ROWS_CF.array());
- Text firstRowQualifier = new Text(Indexer.METRICS_TABLE_FIRST_ROW_CQ.array());
- Text lastRowQualifier = new Text(Indexer.METRICS_TABLE_LAST_ROW_CQ.array());
- scanner.fetchColumn(family, firstRowQualifier);
- scanner.fetchColumn(family, lastRowQualifier);
-
- byte[] firstRow = null;
- byte[] lastRow = null;
- for (Entry entry : scanner) {
- if (entry.getKey().compareColumnQualifier(firstRowQualifier) == 0) {
- firstRow = entry.getValue().get();
- }
-
- if (entry.getKey().compareColumnQualifier(lastRowQualifier) == 0) {
- lastRow = entry.getValue().get();
- }
- }
- scanner.close();
- return Pair.of(firstRow, lastRow);
- }
-
- /**
- * Class containing the key for aggregating the local metrics counter.
- */
- private static class MetricsKey
- {
- private static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility();
-
- public final ByteBuffer row;
- public final ByteBuffer family;
- public final ColumnVisibility visibility;
-
- public MetricsKey(ByteBuffer row, ByteBuffer family)
- {
- requireNonNull(row, "row is null");
- requireNonNull(family, "family is null");
- this.row = row;
- this.family = family;
- this.visibility = EMPTY_VISIBILITY;
- }
-
- public MetricsKey(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility)
- {
- requireNonNull(row, "row is null");
- requireNonNull(family, "family is null");
- requireNonNull(visibility, "visibility is null");
- this.row = row;
- this.family = family;
- this.visibility = visibility.getExpression() != null ? visibility : EMPTY_VISIBILITY;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- MetricsKey other = (MetricsKey) obj;
- return Objects.equals(this.row, other.row)
- && Objects.equals(this.family, other.family)
- && Objects.equals(this.visibility, other.visibility);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(row, family, visibility);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("row", new String(row.array(), UTF_8))
- .add("family", new String(row.array(), UTF_8))
- .add("visibility", visibility.toString())
- .toString();
- }
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSink.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSink.java
deleted file mode 100644
index 56fffe80cb47..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSink.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.io;
-
-import com.google.common.collect.ImmutableList;
-import io.airlift.slice.Slice;
-import io.prestosql.plugin.accumulo.Types;
-import io.prestosql.plugin.accumulo.index.Indexer;
-import io.prestosql.plugin.accumulo.metadata.AccumuloTable;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.Field;
-import io.prestosql.plugin.accumulo.model.Row;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.Page;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ConnectorPageSink;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeUtils;
-import io.prestosql.spi.type.VarcharType;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-
-import static io.airlift.concurrent.MoreFutures.getFutureValue;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static io.prestosql.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR;
-import static io.prestosql.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
-import static io.prestosql.spi.type.BigintType.BIGINT;
-import static io.prestosql.spi.type.BooleanType.BOOLEAN;
-import static io.prestosql.spi.type.DateType.DATE;
-import static io.prestosql.spi.type.DoubleType.DOUBLE;
-import static io.prestosql.spi.type.IntegerType.INTEGER;
-import static io.prestosql.spi.type.RealType.REAL;
-import static io.prestosql.spi.type.SmallintType.SMALLINT;
-import static io.prestosql.spi.type.TimeType.TIME;
-import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
-import static io.prestosql.spi.type.TinyintType.TINYINT;
-import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
-import static java.util.Objects.requireNonNull;
-import static java.util.concurrent.CompletableFuture.completedFuture;
-
-/**
- * Output class for serializing Presto pages (blocks of rows of data) to Accumulo.
- * This class converts the rows from within a page to a collection of Accumulo Mutations,
- * writing and indexed the rows. Writers are flushed and closed on commit, and if a rollback occurs...
- * we'll you're gonna have a bad time.
- *
- * @see AccumuloPageSinkProvider
- */
-public class AccumuloPageSink
- implements ConnectorPageSink
-{
- public static final Text ROW_ID_COLUMN = new Text("___ROW___");
- private final AccumuloRowSerializer serializer;
- private final BatchWriter writer;
- private final Optional indexer;
- private final List columns;
- private final int rowIdOrdinal;
- private long numRows;
-
- public AccumuloPageSink(
- Connector connector,
- AccumuloTable table,
- String username)
- {
- requireNonNull(table, "table is null");
-
- this.columns = table.getColumns();
-
- // Fetch the row ID ordinal, throwing an exception if not found for safety
- Optional ordinal = columns.stream()
- .filter(columnHandle -> columnHandle.getName().equals(table.getRowId()))
- .map(AccumuloColumnHandle::getOrdinal)
- .findAny();
-
- if (!ordinal.isPresent()) {
- throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Row ID ordinal not found");
- }
-
- this.rowIdOrdinal = ordinal.get();
- this.serializer = table.getSerializerInstance();
-
- try {
- // Create a BatchWriter to the Accumulo table
- BatchWriterConfig conf = new BatchWriterConfig();
- writer = connector.createBatchWriter(table.getFullTableName(), conf);
-
- // If the table is indexed, create an instance of an Indexer, else empty
- if (table.isIndexed()) {
- indexer = Optional.of(
- new Indexer(
- connector,
- connector.securityOperations().getUserAuthorizations(username),
- table,
- conf));
- }
- else {
- indexer = Optional.empty();
- }
- }
- catch (AccumuloException | AccumuloSecurityException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Accumulo error when creating BatchWriter and/or Indexer", e);
- }
- catch (TableNotFoundException e) {
- throw new PrestoException(ACCUMULO_TABLE_DNE, "Accumulo error when creating BatchWriter and/or Indexer, table does not exist", e);
- }
- }
-
- /**
- * Converts a {@link Row} to an Accumulo mutation.
- *
- * @param row Row object
- * @param rowIdOrdinal Ordinal in the list of columns that is the row ID. This isn't checked at all, so I hope you're right. Also, it is expected that the list of column handles is sorted in ordinal order. This is a very demanding function.
- * @param columns All column handles for the Row, sorted by ordinal.
- * @param serializer Instance of {@link AccumuloRowSerializer} used to encode the values of the row to the Mutation
- * @return Mutation
- */
- public static Mutation toMutation(Row row, int rowIdOrdinal, List columns, AccumuloRowSerializer serializer)
- {
- // Set our value to the row ID
- Text value = new Text();
- Field rowField = row.getField(rowIdOrdinal);
- if (rowField.isNull()) {
- throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Column mapped as the Accumulo row ID cannot be null");
- }
-
- setText(rowField, value, serializer);
-
- // Iterate through all the column handles, setting the Mutation's columns
- Mutation mutation = new Mutation(value);
-
- // Store row ID in a special column
- mutation.put(ROW_ID_COLUMN, ROW_ID_COLUMN, new Value(value.copyBytes()));
- for (AccumuloColumnHandle columnHandle : columns) {
- // Skip the row ID ordinal
- if (columnHandle.getOrdinal() == rowIdOrdinal) {
- continue;
- }
-
- // If the value of the field is not null
- if (!row.getField(columnHandle.getOrdinal()).isNull()) {
- // Serialize the value to the text
- setText(row.getField(columnHandle.getOrdinal()), value, serializer);
-
- // And add the bytes to the Mutation
- mutation.put(columnHandle.getFamily().get(), columnHandle.getQualifier().get(), new Value(value.copyBytes()));
- }
- }
-
- return mutation;
- }
-
- private static void setText(Field field, Text value, AccumuloRowSerializer serializer)
- {
- Type type = field.getType();
- if (Types.isArrayType(type)) {
- serializer.setArray(value, type, field.getArray());
- }
- else if (Types.isMapType(type)) {
- serializer.setMap(value, type, field.getMap());
- }
- else {
- if (type.equals(BIGINT)) {
- serializer.setLong(value, field.getLong());
- }
- else if (type.equals(BOOLEAN)) {
- serializer.setBoolean(value, field.getBoolean());
- }
- else if (type.equals(DATE)) {
- serializer.setDate(value, field.getDate());
- }
- else if (type.equals(DOUBLE)) {
- serializer.setDouble(value, field.getDouble());
- }
- else if (type.equals(INTEGER)) {
- serializer.setInt(value, field.getInt());
- }
- else if (type.equals(REAL)) {
- serializer.setFloat(value, field.getFloat());
- }
- else if (type.equals(SMALLINT)) {
- serializer.setShort(value, field.getShort());
- }
- else if (type.equals(TIME)) {
- serializer.setTime(value, field.getTime());
- }
- else if (type.equals(TINYINT)) {
- serializer.setByte(value, field.getByte());
- }
- else if (type.equals(TIMESTAMP)) {
- serializer.setTimestamp(value, field.getTimestamp());
- }
- else if (type.equals(VARBINARY)) {
- serializer.setVarbinary(value, field.getVarbinary());
- }
- else if (type instanceof VarcharType) {
- serializer.setVarchar(value, field.getVarchar());
- }
- else {
- throw new UnsupportedOperationException("Unsupported type " + type);
- }
- }
- }
-
- @Override
- public CompletableFuture appendPage(Page page)
- {
- // For each position within the page, i.e. row
- for (int position = 0; position < page.getPositionCount(); ++position) {
- Row row = new Row();
- // For each channel within the page, i.e. column
- for (int channel = 0; channel < page.getChannelCount(); ++channel) {
- // Get the type for this channel
- Type type = columns.get(channel).getType();
-
- // Read the value from the page and append the field to the row
- row.addField(TypeUtils.readNativeValue(type, page.getBlock(channel), position), type);
- }
-
- try {
- // Convert row to a Mutation, writing and indexing it
- Mutation mutation = toMutation(row, rowIdOrdinal, columns, serializer);
- writer.addMutation(mutation);
- if (indexer.isPresent()) {
- indexer.get().index(mutation);
- }
- ++numRows;
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server", e);
- }
-
- // TODO Fix arbitrary flush every 100k rows
- if (numRows % 100_000 == 0) {
- flush();
- }
- }
-
- return NOT_BLOCKED;
- }
-
- @Override
- public CompletableFuture> finish()
- {
- try {
- // Done serializing rows, so flush and close the writer and indexer
- writer.flush();
- writer.close();
- if (indexer.isPresent()) {
- indexer.get().close();
- }
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e);
- }
-
- // TODO Look into any use of the metadata for writing out the rows
- return completedFuture(ImmutableList.of());
- }
-
- @Override
- public void abort()
- {
- getFutureValue(finish());
- }
-
- private void flush()
- {
- try {
- if (indexer.isPresent()) {
- indexer.get().flush();
- // MetricsWriter is non-null if Indexer is present
- }
- writer.flush();
- }
- catch (MutationsRejectedException e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e);
- }
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSinkProvider.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSinkProvider.java
deleted file mode 100644
index 7a75181cd6ac..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloPageSinkProvider.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.io;
-
-import io.prestosql.plugin.accumulo.AccumuloClient;
-import io.prestosql.plugin.accumulo.conf.AccumuloConfig;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.spi.connector.ConnectorInsertTableHandle;
-import io.prestosql.spi.connector.ConnectorOutputTableHandle;
-import io.prestosql.spi.connector.ConnectorPageSink;
-import io.prestosql.spi.connector.ConnectorPageSinkProvider;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-import org.apache.accumulo.core.client.Connector;
-
-import javax.inject.Inject;
-
-import static java.util.Objects.requireNonNull;
-
-/**
- * Page sink provider for Accumulo connector. Creates {@link AccumuloPageSink} objects for output tables (CTAS) and inserts.
- *
- * @see AccumuloPageSink
- */
-public class AccumuloPageSinkProvider
- implements ConnectorPageSinkProvider
-{
- private final AccumuloClient client;
- private final Connector connector;
- private final String username;
-
- @Inject
- public AccumuloPageSinkProvider(
- Connector connector,
- AccumuloConfig config,
- AccumuloClient client)
- {
- this.client = requireNonNull(client, "client is null");
- this.connector = requireNonNull(connector, "connector is null");
- this.username = requireNonNull(config, "config is null").getUsername();
- }
-
- @Override
- public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle)
- {
- AccumuloTableHandle tableHandle = (AccumuloTableHandle) outputTableHandle;
- return new AccumuloPageSink(connector, client.getTable(tableHandle.toSchemaTableName()), username);
- }
-
- @Override
- public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle)
- {
- return createPageSink(transactionHandle, session, (ConnectorOutputTableHandle) insertTableHandle);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordCursor.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordCursor.java
deleted file mode 100644
index 6f1c4fd64b4e..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordCursor.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.io;
-
-import io.airlift.slice.Slice;
-import io.airlift.slice.Slices;
-import io.prestosql.plugin.accumulo.Types;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.RecordCursor;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.VarbinaryType;
-import io.prestosql.spi.type.VarcharType;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.FirstEntryInRowIterator;
-import org.apache.accumulo.core.iterators.user.WholeRowIterator;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.io.Text;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.IO_ERROR;
-import static io.prestosql.plugin.accumulo.io.AccumuloPageSink.ROW_ID_COLUMN;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
-import static io.prestosql.spi.type.BigintType.BIGINT;
-import static io.prestosql.spi.type.BooleanType.BOOLEAN;
-import static io.prestosql.spi.type.DateType.DATE;
-import static io.prestosql.spi.type.DoubleType.DOUBLE;
-import static io.prestosql.spi.type.IntegerType.INTEGER;
-import static io.prestosql.spi.type.RealType.REAL;
-import static io.prestosql.spi.type.SmallintType.SMALLINT;
-import static io.prestosql.spi.type.TimeType.TIME;
-import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
-import static io.prestosql.spi.type.TinyintType.TINYINT;
-import static java.lang.String.format;
-import static java.util.Objects.requireNonNull;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-/**
- * Implementation of Presto RecordCursor, responsible for iterating over a Presto split,
- * reading rows of data and then implementing various methods to retrieve columns within each row.
- *
- * @see AccumuloRecordSet
- * @see AccumuloRecordSetProvider
- */
-public class AccumuloRecordCursor
- implements RecordCursor
-{
- private static final int WHOLE_ROW_ITERATOR_PRIORITY = Integer.MAX_VALUE;
-
- private final List columnHandles;
- private final String[] fieldToColumnName;
- private final BatchScanner scanner;
- private final Iterator> iterator;
- private final AccumuloRowSerializer serializer;
-
- private long bytesRead;
-
- public AccumuloRecordCursor(
- AccumuloRowSerializer serializer,
- BatchScanner scanner,
- String rowIdName,
- List columnHandles)
- {
- this.columnHandles = requireNonNull(columnHandles, "columnHandles is null");
- this.scanner = requireNonNull(scanner, "scanner is null");
- this.serializer = requireNonNull(serializer, "serializer is null");
- this.serializer.setRowIdName(requireNonNull(rowIdName, "rowIdName is null"));
-
- requireNonNull(columnHandles, "columnHandles is null");
-
- if (retrieveOnlyRowIds(rowIdName)) {
- this.scanner.addScanIterator(new IteratorSetting(1, "firstentryiter", FirstEntryInRowIterator.class));
-
- fieldToColumnName = new String[1];
- fieldToColumnName[0] = rowIdName;
-
- // Set a flag on the serializer saying we are only going to be retrieving the row ID
- this.serializer.setRowOnly(true);
- }
- else {
- // Else, we will be scanning some more columns here
- this.serializer.setRowOnly(false);
-
- // Fetch the reserved row ID column
- this.scanner.fetchColumn(ROW_ID_COLUMN, ROW_ID_COLUMN);
-
- Text family = new Text();
- Text qualifier = new Text();
-
- // Create an array which maps the column ordinal to the name of the column
- fieldToColumnName = new String[columnHandles.size()];
- for (int i = 0; i < columnHandles.size(); ++i) {
- AccumuloColumnHandle columnHandle = columnHandles.get(i);
- fieldToColumnName[i] = columnHandle.getName();
-
- // Make sure to skip the row ID!
- if (!columnHandle.getName().equals(rowIdName)) {
- // Set the mapping of presto column name to the family/qualifier
- this.serializer.setMapping(columnHandle.getName(), columnHandle.getFamily().get(), columnHandle.getQualifier().get());
-
- // Set our scanner to fetch this family/qualifier column
- // This will help us prune which data we receive from Accumulo
- family.set(columnHandle.getFamily().get());
- qualifier.set(columnHandle.getQualifier().get());
- this.scanner.fetchColumn(family, qualifier);
- }
- }
- }
-
- IteratorSetting setting = new IteratorSetting(WHOLE_ROW_ITERATOR_PRIORITY, WholeRowIterator.class);
- scanner.addScanIterator(setting);
-
- iterator = this.scanner.iterator();
- }
-
- @Override
- public long getCompletedBytes()
- {
- return bytesRead;
- }
-
- @Override
- public long getReadTimeNanos()
- {
- return 0;
- }
-
- @Override
- public Type getType(int field)
- {
- checkArgument(field >= 0 && field < columnHandles.size(), "Invalid field index");
- return columnHandles.get(field).getType();
- }
-
- @Override
- public boolean advanceNextPosition()
- {
- try {
- if (iterator.hasNext()) {
- serializer.reset();
- Entry row = iterator.next();
- for (Entry entry : WholeRowIterator.decodeRow(row.getKey(), row.getValue()).entrySet()) {
- bytesRead += entry.getKey().getSize() + entry.getValue().getSize();
- serializer.deserialize(entry);
- }
- return true;
- }
- else {
- return false;
- }
- }
- catch (IOException e) {
- throw new PrestoException(IO_ERROR, "Caught IO error from serializer on read", e);
- }
- }
-
- @Override
- public boolean isNull(int field)
- {
- checkArgument(field < columnHandles.size(), "Invalid field index");
- return serializer.isNull(fieldToColumnName[field]);
- }
-
- @Override
- public boolean getBoolean(int field)
- {
- checkFieldType(field, BOOLEAN);
- return serializer.getBoolean(fieldToColumnName[field]);
- }
-
- @Override
- public double getDouble(int field)
- {
- checkFieldType(field, DOUBLE);
- return serializer.getDouble(fieldToColumnName[field]);
- }
-
- @Override
- public long getLong(int field)
- {
- checkFieldType(field, BIGINT, DATE, INTEGER, REAL, SMALLINT, TIME, TIMESTAMP, TINYINT);
- Type type = getType(field);
- if (type.equals(BIGINT)) {
- return serializer.getLong(fieldToColumnName[field]);
- }
- else if (type.equals(DATE)) {
- return MILLISECONDS.toDays(serializer.getDate(fieldToColumnName[field]).getTime());
- }
- else if (type.equals(INTEGER)) {
- return serializer.getInt(fieldToColumnName[field]);
- }
- else if (type.equals(REAL)) {
- return Float.floatToIntBits(serializer.getFloat(fieldToColumnName[field]));
- }
- else if (type.equals(SMALLINT)) {
- return serializer.getShort(fieldToColumnName[field]);
- }
- else if (type.equals(TIME)) {
- return serializer.getTime(fieldToColumnName[field]).getTime();
- }
- else if (type.equals(TIMESTAMP)) {
- return serializer.getTimestamp(fieldToColumnName[field]).getTime();
- }
- else if (type.equals(TINYINT)) {
- return serializer.getByte(fieldToColumnName[field]);
- }
- else {
- throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + getType(field));
- }
- }
-
- @Override
- public Object getObject(int field)
- {
- Type type = getType(field);
- checkArgument(Types.isArrayType(type) || Types.isMapType(type), "Expected field %s to be a type of array or map but is %s", field, type);
-
- if (Types.isArrayType(type)) {
- return serializer.getArray(fieldToColumnName[field], type);
- }
-
- return serializer.getMap(fieldToColumnName[field], type);
- }
-
- @Override
- public Slice getSlice(int field)
- {
- Type type = getType(field);
- if (type instanceof VarbinaryType) {
- return Slices.wrappedBuffer(serializer.getVarbinary(fieldToColumnName[field]));
- }
- else if (type instanceof VarcharType) {
- return Slices.utf8Slice(serializer.getVarchar(fieldToColumnName[field]));
- }
- else {
- throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + type);
- }
- }
-
- @Override
- public void close()
- {
- scanner.close();
- }
-
- /**
- * Gets a Boolean value indicating whether or not the scanner should only return row IDs.
- *
- * This can occur in cases such as SELECT COUNT(*) or the table only has one column.
- * Presto doesn't need the entire contents of the row to count them,
- * so we can configure Accumulo to only give us the first key/value pair in the row
- *
- * @param rowIdName Row ID column name
- * @return True if scanner should retriev eonly row IDs, false otherwise
- */
- private boolean retrieveOnlyRowIds(String rowIdName)
- {
- return columnHandles.isEmpty() || (columnHandles.size() == 1 && columnHandles.get(0).getName().equals(rowIdName));
- }
-
- /**
- * Checks that the given field is one of the provided types.
- *
- * @param field Ordinal of the field
- * @param expected An array of expected types
- * @throws IllegalArgumentException If the given field does not match one of the types
- */
- private void checkFieldType(int field, Type... expected)
- {
- Type actual = getType(field);
- for (Type type : expected) {
- if (actual.equals(type)) {
- return;
- }
- }
-
- throw new IllegalArgumentException(format("Expected field %s to be a type of %s but is %s", field, StringUtils.join(expected, ","), actual));
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSet.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSet.java
deleted file mode 100644
index 82cc65bca8fe..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSet.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.io;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import io.airlift.log.Logger;
-import io.prestosql.plugin.accumulo.conf.AccumuloSessionProperties;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.AccumuloSplit;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.RecordCursor;
-import io.prestosql.spi.connector.RecordSet;
-import io.prestosql.spi.type.Type;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.security.Authorizations;
-
-import java.util.List;
-import java.util.Optional;
-
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
-import static java.lang.String.format;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Implementation of a Presto RecordSet, responsible for returning the column types and the RecordCursor to the framework.
- *
- * @see AccumuloRecordCursor
- * @see AccumuloRecordSetProvider
- */
-public class AccumuloRecordSet
- implements RecordSet
-{
- private static final Logger LOG = Logger.get(AccumuloRecordSet.class);
- private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults();
-
- private final List columnHandles;
- private final List columnTypes;
- private final AccumuloRowSerializer serializer;
- private final BatchScanner scanner;
- private final String rowIdName;
-
- public AccumuloRecordSet(
- Connector connector,
- ConnectorSession session,
- AccumuloSplit split,
- String username,
- AccumuloTableHandle table,
- List columnHandles)
- {
- requireNonNull(session, "session is null");
- requireNonNull(split, "split is null");
- requireNonNull(username, "username is null");
- requireNonNull(table, "table is null");
-
- rowIdName = table.getRowId();
-
- serializer = table.getSerializerInstance();
-
- // Save off the column handles and createa list of the Accumulo types
- this.columnHandles = requireNonNull(columnHandles, "column handles is null");
- ImmutableList.Builder types = ImmutableList.builder();
- for (AccumuloColumnHandle column : columnHandles) {
- types.add(column.getType());
- }
- this.columnTypes = types.build();
-
- try {
- // Create the BatchScanner and set the ranges from the split
- scanner = connector.createBatchScanner(table.getFullTableName(), getScanAuthorizations(session, table, connector, username), 10);
- scanner.setRanges(split.getRanges());
- }
- catch (Exception e) {
- throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, format("Failed to create batch scanner for table %s", table.getFullTableName()), e);
- }
- }
-
- /**
- * Gets the scanner authorizations to use for scanning tables.
- *
- * In order of priority: session username authorizations, then table property, then the default connector auths.
- *
- * @param session Current session
- * @param table Accumulo table
- * @param connector Accumulo connector
- * @param username Accumulo username
- * @return Scan authorizations
- * @throws AccumuloException If a generic Accumulo error occurs
- * @throws AccumuloSecurityException If a security exception occurs
- */
- private static Authorizations getScanAuthorizations(ConnectorSession session, AccumuloTableHandle table, Connector connector, String username)
- throws AccumuloException, AccumuloSecurityException
- {
- String sessionScanUser = AccumuloSessionProperties.getScanUsername(session);
- if (sessionScanUser != null) {
- Authorizations scanAuths = connector.securityOperations().getUserAuthorizations(sessionScanUser);
- LOG.debug("Using session scanner auths for user %s: %s", sessionScanUser, scanAuths);
- return scanAuths;
- }
-
- Optional scanAuths = table.getScanAuthorizations();
- if (scanAuths.isPresent()) {
- Authorizations auths = new Authorizations(Iterables.toArray(COMMA_SPLITTER.split(scanAuths.get()), String.class));
- LOG.debug("scan_auths table property set: %s", auths);
- return auths;
- }
- else {
- Authorizations auths = connector.securityOperations().getUserAuthorizations(username);
- LOG.debug("scan_auths table property not set, using user auths: %s", auths);
- return auths;
- }
- }
-
- @Override
- public List getColumnTypes()
- {
- return columnTypes;
- }
-
- @Override
- public RecordCursor cursor()
- {
- return new AccumuloRecordCursor(serializer, scanner, rowIdName, columnHandles);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSetProvider.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSetProvider.java
deleted file mode 100644
index f71ddbf953df..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/io/AccumuloRecordSetProvider.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.io;
-
-import io.prestosql.plugin.accumulo.conf.AccumuloConfig;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.model.AccumuloSplit;
-import io.prestosql.plugin.accumulo.model.AccumuloTableHandle;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ConnectorRecordSetProvider;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorSplit;
-import io.prestosql.spi.connector.ConnectorTableHandle;
-import io.prestosql.spi.connector.ConnectorTransactionHandle;
-import io.prestosql.spi.connector.RecordSet;
-import org.apache.accumulo.core.client.Connector;
-
-import javax.inject.Inject;
-
-import java.util.List;
-
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static java.util.Objects.requireNonNull;
-
-/**
- * Implementation of a ConnectorRecordSetProvider for Accumulo. Generates {@link AccumuloRecordSet} objects for a provided split.
- *
- * @see AccumuloRecordSet
- * @see AccumuloRecordCursor
- */
-public class AccumuloRecordSetProvider
- implements ConnectorRecordSetProvider
-{
- private final Connector connector;
- private final String username;
-
- @Inject
- public AccumuloRecordSetProvider(
- Connector connector,
- AccumuloConfig config)
- {
- this.connector = requireNonNull(connector, "connector is null");
- this.username = requireNonNull(config, "config is null").getUsername();
- }
-
- @Override
- public RecordSet getRecordSet(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List columns)
- {
- AccumuloSplit accSplit = (AccumuloSplit) split;
- AccumuloTableHandle accTable = (AccumuloTableHandle) table;
-
- List accColumns = columns.stream()
- .map(AccumuloColumnHandle.class::cast)
- .collect(toImmutableList());
-
- return new AccumuloRecordSet(connector, session, accSplit, username, accTable, accColumns);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MaxByteArrayCombiner.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MaxByteArrayCombiner.java
deleted file mode 100644
index d171a35cf99c..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MaxByteArrayCombiner.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.iterators;
-
-import com.google.common.primitives.UnsignedBytes;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.Combiner;
-
-import java.util.Comparator;
-import java.util.Iterator;
-
-/**
- * A Combiner that does a lexicographic compare against values, returning the 'largest' value
- */
-public class MaxByteArrayCombiner
- extends Combiner
-{
- private final Comparator comparator = UnsignedBytes.lexicographicalComparator();
-
- @Override
- public Value reduce(Key key, Iterator iter)
- {
- Value max = null;
- while (iter.hasNext()) {
- Value test = iter.next();
- if (max == null) {
- max = new Value(test.get());
- }
- else if (comparator.compare(test.get(), max.get()) > 0) {
- max.set(test.get());
- }
- }
- return max;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MinByteArrayCombiner.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MinByteArrayCombiner.java
deleted file mode 100644
index 517540d6a6d8..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/iterators/MinByteArrayCombiner.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.iterators;
-
-import com.google.common.primitives.UnsignedBytes;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.Combiner;
-
-import java.util.Comparator;
-import java.util.Iterator;
-
-/**
- * A Combiner that does a lexicographic compare against values, returning the 'smallest' value
- */
-public class MinByteArrayCombiner
- extends Combiner
-{
- private final Comparator comparator = UnsignedBytes.lexicographicalComparator();
-
- @Override
- public Value reduce(Key key, Iterator iter)
- {
- Value min = null;
- while (iter.hasNext()) {
- Value test = iter.next();
- if (min == null) {
- min = new Value(test.get());
- }
- else if (comparator.compare(test.get(), min.get()) < 0) {
- min.set(test.get());
- }
- }
- return min;
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloTable.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloTable.java
deleted file mode 100644
index c530b4e609be..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloTable.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.metadata;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableList;
-import io.prestosql.plugin.accumulo.index.Indexer;
-import io.prestosql.plugin.accumulo.model.AccumuloColumnHandle;
-import io.prestosql.plugin.accumulo.serializers.AccumuloRowSerializer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ColumnMetadata;
-import io.prestosql.spi.connector.SchemaTableName;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.List;
-import java.util.Optional;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static io.prestosql.spi.StandardErrorCode.NOT_FOUND;
-import static java.util.Objects.requireNonNull;
-
-/**
- * This class encapsulates metadata regarding an Accumulo table in Presto.
- */
-public class AccumuloTable
-{
- private final boolean external;
- private final Integer rowIdOrdinal;
- private final String schema;
- private final String serializerClassName;
- private final Optional scanAuthorizations;
- private final List columnsMetadata;
- private final boolean indexed;
- private final List columns;
- private final String rowId;
- private final String table;
- private final SchemaTableName schemaTableName;
-
- @JsonCreator
- public AccumuloTable(
- @JsonProperty("schema") String schema,
- @JsonProperty("table") String table,
- @JsonProperty("columns") List columns,
- @JsonProperty("rowId") String rowId,
- @JsonProperty("external") boolean external,
- @JsonProperty("serializerClassName") String serializerClassName,
- @JsonProperty("scanAuthorizations") Optional scanAuthorizations)
- {
- this.external = external;
- this.rowId = requireNonNull(rowId, "rowId is null");
- this.schema = requireNonNull(schema, "schema is null");
- this.table = requireNonNull(table, "table is null");
- this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns are null"));
- this.serializerClassName = requireNonNull(serializerClassName, "serializerClassName is null");
- this.scanAuthorizations = scanAuthorizations;
-
- boolean indexed = false;
- Optional rowIdOrdinal = Optional.empty();
-
- // Extract the ColumnMetadata from the handles for faster access
- ImmutableList.Builder columnMetadataBuilder = ImmutableList.builder();
- for (AccumuloColumnHandle column : this.columns) {
- columnMetadataBuilder.add(column.getColumnMetadata());
- indexed |= column.isIndexed();
- if (column.getName().equals(this.rowId)) {
- rowIdOrdinal = Optional.of(column.getOrdinal());
- }
- }
-
- if (rowIdOrdinal.isPresent()) {
- this.rowIdOrdinal = rowIdOrdinal.get();
- }
- else {
- throw new IllegalArgumentException("rowIdOrdinal is null, enable to locate rowId in given column list");
- }
-
- this.indexed = indexed;
- this.columnsMetadata = columnMetadataBuilder.build();
- this.schemaTableName = new SchemaTableName(this.schema, this.table);
- }
-
- @JsonProperty
- public String getRowId()
- {
- return rowId;
- }
-
- @JsonProperty
- public String getSchema()
- {
- return schema;
- }
-
- @JsonProperty
- public String getTable()
- {
- return table;
- }
-
- @JsonIgnore
- public String getIndexTableName()
- {
- return Indexer.getIndexTableName(schema, table);
- }
-
- @JsonIgnore
- public String getMetricsTableName()
- {
- return Indexer.getMetricsTableName(schema, table);
- }
-
- @JsonIgnore
- public String getFullTableName()
- {
- return getFullTableName(schema, table);
- }
-
- @JsonProperty
- public List getColumns()
- {
- return columns;
- }
-
- @JsonProperty
- public Optional getScanAuthorizations()
- {
- return scanAuthorizations;
- }
-
- @JsonProperty
- public String getSerializerClassName()
- {
- return serializerClassName;
- }
-
- @JsonIgnore
- public List getColumnsMetadata()
- {
- return columnsMetadata;
- }
-
- @JsonProperty
- public boolean isExternal()
- {
- return external;
- }
-
- @JsonIgnore
- public boolean isIndexed()
- {
- return indexed;
- }
-
- @JsonIgnore
- public int getRowIdOrdinal()
- {
- return this.rowIdOrdinal;
- }
-
- @JsonIgnore
- public AccumuloRowSerializer getSerializerInstance()
- {
- try {
- return (AccumuloRowSerializer) Class.forName(serializerClassName).getConstructor().newInstance();
- }
- catch (ClassNotFoundException | InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
- throw new PrestoException(NOT_FOUND, "Configured serializer class not found", e);
- }
- }
-
- @JsonIgnore
- public static String getFullTableName(String schema, String table)
- {
- return schema.equals("default") ? table : schema + '.' + table;
- }
-
- @JsonIgnore
- public static String getFullTableName(SchemaTableName tableName)
- {
- return getFullTableName(tableName.getSchemaName(), tableName.getTableName());
- }
-
- @JsonIgnore
- public SchemaTableName getSchemaTableName()
- {
- return schemaTableName;
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("schemaName", schema)
- .add("tableName", table)
- .add("columns", columns)
- .add("rowIdName", rowId)
- .add("external", external)
- .add("serializerClassName", serializerClassName)
- .add("scanAuthorizations", scanAuthorizations)
- .toString();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloView.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloView.java
deleted file mode 100644
index 551e9449524f..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/AccumuloView.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.metadata;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.prestosql.spi.connector.SchemaTableName;
-
-import java.util.Objects;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static java.util.Objects.requireNonNull;
-
-/**
- * This class encapsulates metadata regarding an Accumulo view in Presto.
- */
-public class AccumuloView
-{
- private final String schema;
- private final String table;
- private final String data;
- private final SchemaTableName schemaTableName;
-
- @JsonCreator
- public AccumuloView(
- @JsonProperty("schema") String schema,
- @JsonProperty("table") String table,
- @JsonProperty("data") String data)
- {
- this.schema = requireNonNull(schema, "schema is null");
- this.table = requireNonNull(table, "table is null");
- this.data = requireNonNull(data, "data is null");
- this.schemaTableName = new SchemaTableName(schema, table);
- }
-
- @JsonProperty
- public String getSchema()
- {
- return schema;
- }
-
- @JsonProperty
- public String getTable()
- {
- return table;
- }
-
- @JsonProperty
- public String getData()
- {
- return data;
- }
-
- @JsonIgnore
- public SchemaTableName getSchemaTableName()
- {
- return schemaTableName;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(schema, table, data);
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- AccumuloView other = (AccumuloView) obj;
- return Objects.equals(this.schema, other.schema)
- && Objects.equals(this.table, other.table)
- && Objects.equals(this.data, other.data);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("schema", schema)
- .add("table", table)
- .add("data", data)
- .toString();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/ZooKeeperMetadataManager.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/ZooKeeperMetadataManager.java
deleted file mode 100644
index e582cadc32e6..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/metadata/ZooKeeperMetadataManager.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.metadata;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.airlift.json.ObjectMapperProvider;
-import io.prestosql.plugin.accumulo.AccumuloModule;
-import io.prestosql.plugin.accumulo.conf.AccumuloConfig;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeManager;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryForever;
-import org.apache.zookeeper.KeeperException;
-
-import javax.inject.Inject;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Set;
-
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static io.prestosql.plugin.accumulo.AccumuloErrorCode.ZOOKEEPER_ERROR;
-import static java.lang.String.format;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
-import static org.apache.zookeeper.KeeperException.Code.NONODE;
-
-public class ZooKeeperMetadataManager
-{
- private static final String DEFAULT_SCHEMA = "default";
-
- private final CuratorFramework curator;
- private final ObjectMapper mapper;
-
- @Inject
- public ZooKeeperMetadataManager(AccumuloConfig config, TypeManager typeManager)
- {
- requireNonNull(typeManager, "typeManager is null");
-
- // Create JSON deserializer for the AccumuloTable
- ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider();
- objectMapperProvider.setJsonDeserializers(ImmutableMap.of(Type.class, new AccumuloModule.TypeDeserializer(typeManager)));
- mapper = objectMapperProvider.get();
-
- String zkMetadataRoot = config.getZkMetadataRoot();
- String zookeepers = config.getZooKeepers();
-
- // Create the connection to ZooKeeper to check if the metadata root exists
- CuratorFramework checkRoot = CuratorFrameworkFactory.newClient(zookeepers, new RetryForever(1000));
- checkRoot.start();
-
- try {
- // If the metadata root does not exist, create it
- if (checkRoot.checkExists().forPath(zkMetadataRoot) == null) {
- checkRoot.create().forPath(zkMetadataRoot);
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error checking metadata root", e);
- }
- checkRoot.close();
-
- // Create the curator client framework to use for metadata management, set at the ZK root
- curator = CuratorFrameworkFactory.newClient(zookeepers + zkMetadataRoot, new RetryForever(1000));
- curator.start();
-
- try {
- // Create default schema should it not exist
- if (curator.checkExists().forPath("/" + DEFAULT_SCHEMA) == null) {
- curator.create().forPath("/" + DEFAULT_SCHEMA);
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error checking/creating default schema", e);
- }
- }
-
- public Set getSchemaNames()
- {
- try {
- Set schemas = new HashSet<>();
- schemas.addAll(curator.getChildren().forPath("/"));
- return schemas;
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error fetching schemas", e);
- }
- }
-
- public Set getTableNames(String schema)
- {
- String schemaPath = getSchemaPath(schema);
- try {
- if (curator.checkExists().forPath(schemaPath) == null) {
- return ImmutableSet.of();
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if schema exists", e);
- }
-
- try {
- return curator.getChildren().forPath(schemaPath).stream()
- .filter(x -> isAccumuloTable(new SchemaTableName(schema, x)))
- .collect(toImmutableSet());
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error fetching schemas", e);
- }
- }
-
- public AccumuloTable getTable(SchemaTableName stName)
- {
- try {
- if (curator.checkExists().forPath(getTablePath(stName)) != null) {
- return toAccumuloTable(curator.getData().forPath(getTablePath(stName)));
- }
-
- return null;
- }
- catch (Exception e) {
- // Capture race condition between checkExists and getData
- if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
- return null;
- }
-
- throw new PrestoException(ZOOKEEPER_ERROR, "Error fetching table", e);
- }
- }
-
- public Set getViewNames(String schema)
- {
- String schemaPath = getSchemaPath(schema);
- try {
- if (curator.checkExists().forPath(schemaPath) == null) {
- return ImmutableSet.of();
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if schema exists", e);
- }
-
- try {
- return curator.getChildren().forPath(schemaPath).stream()
- .filter(x -> isAccumuloView(new SchemaTableName(schema, x)))
- .collect(toImmutableSet());
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error fetching schemas", e);
- }
- }
-
- public AccumuloView getView(SchemaTableName stName)
- {
- try {
- String tablePath = getTablePath(stName);
- if (curator.checkExists().forPath(tablePath) != null) {
- byte[] data = curator.getData().forPath(tablePath);
- if (isAccumuloView(data)) {
- return toAccumuloView(data);
- }
- }
-
- return null;
- }
- catch (Exception e) {
- // Capture race condition between checkExists and getData
- if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
- return null;
- }
-
- throw new PrestoException(ZOOKEEPER_ERROR, "Error fetching view", e);
- }
- }
-
- public void createTableMetadata(AccumuloTable table)
- {
- SchemaTableName tableName = table.getSchemaTableName();
- String tablePath = getTablePath(tableName);
- try {
- if (curator.checkExists().forPath(tablePath) != null) {
- throw new IOException(format("Metadata for table %s already exists", tableName));
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error when checking if table already exists", e);
- }
-
- try {
- curator.create().creatingParentsIfNeeded().forPath(tablePath, toJsonBytes(table));
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error creating table znode in ZooKeeper", e);
- }
- }
-
- public void deleteTableMetadata(SchemaTableName tableName)
- {
- try {
- curator.delete().deletingChildrenIfNeeded().forPath(getTablePath(tableName));
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error when deleting table metadata", e);
- }
- }
-
- public void createViewMetadata(AccumuloView view)
- {
- SchemaTableName tableName = view.getSchemaTableName();
- String viewPath = getTablePath(tableName);
- try {
- if (curator.checkExists().forPath(viewPath) != null) {
- throw new IOException(format("Metadata for view %s already exists", tableName));
- }
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error when checking if view already exists", e);
- }
-
- try {
- curator.create().creatingParentsIfNeeded().forPath(viewPath, toJsonBytes(view));
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "Error creating view znode in ZooKeeper", e);
- }
- }
-
- public void deleteViewMetadata(SchemaTableName tableName)
- {
- try {
- curator.delete().deletingChildrenIfNeeded().forPath(getTablePath(tableName));
- }
- catch (Exception e) {
- throw new PrestoException(ZOOKEEPER_ERROR, "ZK error when deleting view metadata", e);
- }
- }
-
- private static String getSchemaPath(String schema)
- {
- return "/" + schema.toLowerCase(Locale.ENGLISH);
- }
-
- private static String getSchemaPath(SchemaTableName tableName)
- {
- return getSchemaPath(tableName.getSchemaName());
- }
-
- private static String getTablePath(SchemaTableName tableName)
- {
- return getSchemaPath(tableName) + '/' + tableName.getTableName().toLowerCase(Locale.ENGLISH);
- }
-
- private boolean isAccumuloTable(SchemaTableName tableName)
- {
- try {
- String path = getTablePath(tableName);
- return curator.checkExists().forPath(path) != null && isAccumuloTable(curator.getData().forPath(path));
- }
- catch (Exception e) {
- // Capture race condition between checkExists and getData
- if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
- return false;
- }
-
- throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if path %s is an AccumuloTable object", e);
- }
- }
-
- private boolean isAccumuloView(SchemaTableName tableName)
- {
- try {
- String path = getTablePath(tableName);
- return curator.checkExists().forPath(path) != null && isAccumuloView(curator.getData().forPath(path));
- }
- catch (Exception e) {
- // Capture race condition between checkExists and getData
- if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) {
- return false;
- }
-
- throw new PrestoException(ZOOKEEPER_ERROR, "Error checking if path is an AccumuloView object", e);
- }
- }
-
- private boolean isAccumuloTable(byte[] data)
- throws IOException
- {
- // AccumuloTable does not contain a 'data' node
- return !mapper.reader().readTree(new String(data, UTF_8)).has("data");
- }
-
- private boolean isAccumuloView(byte[] data)
- throws IOException
- {
- // AccumuloView contains a 'data' node
- return mapper.reader().readTree(new String(data, UTF_8)).has("data");
- }
-
- private AccumuloTable toAccumuloTable(byte[] data)
- throws IOException
- {
- return mapper.readValue(new String(data, UTF_8), AccumuloTable.class);
- }
-
- private AccumuloView toAccumuloView(byte[] data)
- throws IOException
- {
- return mapper.readValue(new String(data, UTF_8), AccumuloView.class);
- }
-
- private byte[] toJsonBytes(Object obj)
- throws IOException
- {
- return mapper.writeValueAsBytes(obj);
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnConstraint.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnConstraint.java
deleted file mode 100644
index 7f4100982415..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnConstraint.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.model;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.prestosql.spi.predicate.Domain;
-
-import java.util.Objects;
-import java.util.Optional;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloColumnConstraint
-{
- private final String name;
- private final String family;
- private final String qualifier;
- private final boolean indexed;
- private final Optional domain;
-
- @JsonCreator
- public AccumuloColumnConstraint(
- @JsonProperty("name") String name,
- @JsonProperty("family") String family,
- @JsonProperty("qualifier") String qualifier,
- @JsonProperty("domain") Optional domain,
- @JsonProperty("indexed") boolean indexed)
- {
- this.name = requireNonNull(name, "name is null");
- this.family = requireNonNull(family, "family is null");
- this.qualifier = requireNonNull(qualifier, "qualifier is null");
- this.indexed = indexed;
- this.domain = requireNonNull(domain, "domain is null");
- }
-
- @JsonProperty
- public boolean isIndexed()
- {
- return indexed;
- }
-
- @JsonProperty
- public String getName()
- {
- return name;
- }
-
- @JsonProperty
- public String getFamily()
- {
- return family;
- }
-
- @JsonProperty
- public String getQualifier()
- {
- return qualifier;
- }
-
- @JsonProperty
- public Optional getDomain()
- {
- return domain;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(name, family, qualifier, domain, indexed);
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
-
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- AccumuloColumnConstraint other = (AccumuloColumnConstraint) obj;
- return Objects.equals(this.name, other.name)
- && Objects.equals(this.family, other.family)
- && Objects.equals(this.qualifier, other.qualifier)
- && Objects.equals(this.domain, other.domain)
- && Objects.equals(this.indexed, other.indexed);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("name", this.name)
- .add("family", this.family)
- .add("qualifier", this.qualifier)
- .add("indexed", this.indexed)
- .add("domain", this.domain)
- .toString();
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnHandle.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnHandle.java
deleted file mode 100644
index 58e11f37079f..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloColumnHandle.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.model;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ColumnMetadata;
-import io.prestosql.spi.type.Type;
-
-import java.util.Objects;
-import java.util.Optional;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.util.Objects.requireNonNull;
-
-public final class AccumuloColumnHandle
- implements ColumnHandle, Comparable
-{
- private final boolean indexed;
- private final Optional family;
- private final Optional qualifier;
- private final Type type;
- private final String comment;
- private final String name;
- private final int ordinal;
-
- @JsonCreator
- public AccumuloColumnHandle(
- @JsonProperty("name") String name,
- @JsonProperty("family") Optional family,
- @JsonProperty("qualifier") Optional qualifier,
- @JsonProperty("type") Type type,
- @JsonProperty("ordinal") int ordinal,
- @JsonProperty("comment") String comment,
- @JsonProperty("indexed") boolean indexed)
- {
- this.name = requireNonNull(name, "name is null");
- this.family = requireNonNull(family, "family is null");
- this.qualifier = requireNonNull(qualifier, "qualifier is null");
- this.type = requireNonNull(type, "type is null");
- checkArgument(ordinal >= 0, "ordinal must be >= zero");
- this.ordinal = ordinal;
-
- this.comment = requireNonNull(comment, "comment is null");
- this.indexed = indexed;
- }
-
- @JsonProperty
- public String getName()
- {
- return name;
- }
-
- @JsonProperty
- public Optional getFamily()
- {
- return family;
- }
-
- @JsonProperty
- public Optional getQualifier()
- {
- return qualifier;
- }
-
- @JsonProperty
- public Type getType()
- {
- return type;
- }
-
- @JsonProperty
- public int getOrdinal()
- {
- return ordinal;
- }
-
- @JsonProperty
- public String getComment()
- {
- return comment;
- }
-
- @JsonIgnore
- public ColumnMetadata getColumnMetadata()
- {
- return ColumnMetadata.builder()
- .setName(name)
- .setType(type)
- .setComment(Optional.ofNullable(comment))
- .build();
- }
-
- @JsonProperty
- public boolean isIndexed()
- {
- return indexed;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(indexed, name, family, qualifier, type, ordinal, comment);
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
-
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
-
- AccumuloColumnHandle other = (AccumuloColumnHandle) obj;
- return Objects.equals(this.indexed, other.indexed)
- && Objects.equals(this.name, other.name)
- && Objects.equals(this.family, other.family)
- && Objects.equals(this.qualifier, other.qualifier)
- && Objects.equals(this.type, other.type)
- && Objects.equals(this.ordinal, other.ordinal)
- && Objects.equals(this.comment, other.comment);
- }
-
- @Override
- public String toString()
- {
- return toStringHelper(this)
- .add("name", name)
- .add("columnFamily", family.orElse(null))
- .add("columnQualifier", qualifier.orElse(null))
- .add("type", type)
- .add("ordinal", ordinal)
- .add("comment", comment)
- .add("indexed", indexed)
- .toString();
- }
-
- @Override
- public int compareTo(AccumuloColumnHandle obj)
- {
- return Integer.compare(this.getOrdinal(), obj.getOrdinal());
- }
-}
diff --git a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloSplit.java b/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloSplit.java
deleted file mode 100644
index f6feb0d040b3..000000000000
--- a/presto-accumulo/src/main/java/io/prestosql/plugin/accumulo/model/AccumuloSplit.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.prestosql.plugin.accumulo.model;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableList;
-import io.prestosql.spi.HostAddress;
-import io.prestosql.spi.connector.ConnectorSplit;
-import org.apache.accumulo.core.data.Range;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static com.google.common.base.MoreObjects.toStringHelper;
-import static java.util.Objects.requireNonNull;
-
-public class AccumuloSplit
- implements ConnectorSplit
-{
- private final Optional hostPort;
- private final List addresses;
- private final List ranges;
-
- @JsonCreator
- public AccumuloSplit(
- @JsonProperty("ranges") List ranges,
- @JsonProperty("hostPort") Optional hostPort)
- {
- this.hostPort = requireNonNull(hostPort, "hostPort is null");
- this.ranges = ImmutableList.copyOf(requireNonNull(ranges, "ranges is null"));
-
- // Parse the host address into a list of addresses, this would be an Accumulo Tablet server or some localhost thing
- if (hostPort.isPresent()) {
- addresses = ImmutableList.of(HostAddress.fromString(hostPort.get()));
- }
- else {
- addresses = ImmutableList.of();
- }
- }
-
- @JsonProperty
- public Optional getHostPort()
- {
- return hostPort;
- }
-
- @JsonProperty("ranges")
- public List getWrappedRanges()
- {
- return ranges;
- }
-
- @JsonIgnore
- public List getRanges()
- {
- return ranges.stream().map(WrappedRange::getRange).collect(Collectors.toList());
- }
-
- @Override
- public boolean isRemotelyAccessible()
- {
- return true;
- }
-
- @Override
- public List