diff --git a/direct/io-bigtable/pom.xml b/direct/io-bigtable/pom.xml index 0239c339b..4198dfd04 100644 --- a/direct/io-bigtable/pom.xml +++ b/direct/io-bigtable/pom.xml @@ -33,8 +33,8 @@ cz.o2.proxima.bigtable.shaded - 1.12.0 - 1.21.0 + 1.19.0 + 1.33.1 @@ -219,7 +219,7 @@ com.google.cloud.bigtable - bigtable-hbase-1.x + bigtable-hbase-2.x ${bigtable.version} diff --git a/direct/io-hbase-bindings/pom.xml b/direct/io-hbase-bindings/pom.xml index a5b5e43e3..1b23ef025 100644 --- a/direct/io-hbase-bindings/pom.xml +++ b/direct/io-hbase-bindings/pom.xml @@ -179,6 +179,12 @@ ${guava.version} + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + + org.apache.hadoop hadoop-common @@ -206,6 +212,34 @@ + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + test + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + test + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop.version} + test + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + test + + org.apache.hadoop hadoop-hdfs diff --git a/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java b/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java index b64a22ba3..d5b42fc52 100644 --- a/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java +++ b/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -45,24 +44,17 @@ class HBaseWriter extends HBaseClientWrapper implements OnlineAttributeWriter { private static final String DEL_BATCH_SIZE_CONF = "del-batch-size"; - private static final String FLUSH_COMMITS_CFG = "flush-commits"; private final int batchSize; private final Map cfg; private final InternalSerializer serializer; - private boolean flushCommits; - HBaseWriter(URI uri, Configuration conf, Map cfg) { super(uri, conf); batchSize = Optional.ofNullable(cfg.get(DEL_BATCH_SIZE_CONF)) .map(o -> Integer.valueOf(o.toString())) .orElse(1000); - flushCommits = - Optional.ofNullable(cfg.get(FLUSH_COMMITS_CFG)) - .map(o -> Boolean.valueOf(o.toString())) - .orElse(true); serializer = HBaseDataAccessor.instantiateSerializer(uri); this.cfg = cfg; } @@ -89,9 +81,6 @@ public void write(StreamElement data, CommitCallback statusCallback) { Put put = serializer.toPut(family, key, data); this.client.put(put); } - if (flushCommits) { - ((HTable) this.client).flushCommits(); - } statusCallback.commit(true, null); } catch (Exception ex) { log.error("Failed to write {}", data, ex); @@ -148,12 +137,4 @@ private void deletePrefix(byte[] key, byte[] family, String prefix, long stamp) stamp); } } - - @Override - void ensureClient() { - super.ensureClient(); - if (!(client instanceof HTable)) { - flushCommits = false; - } - } } diff --git a/direct/pom.xml b/direct/pom.xml index 3556c066b..662e3a99f 100644 --- a/direct/pom.xml +++ b/direct/pom.xml @@ -66,63 +66,5 @@ - - - jdk8 - - 1.8 - - - compiler - core - core-testing - ingest-client - ingest-server - io-bigtable - io-blob - io-bulkfs - io-bulkfs-parquet - io-cassandra - io-gcloud-storage - io-hadoop - io-hbase - io-hbase-bindings - io-hdfs - io-http - io-kafka - io-pubsub - io-s3 - rpc - transaction-manager - - - - - jdk11 - - 11 - - - compiler - core - core-testing - ingest-client - ingest-server - io-bigtable - io-blob - io-bulkfs - io-bulkfs-parquet - io-cassandra - io-gcloud-storage - io-http - io-kafka - io-pubsub - io-s3 - rpc - transaction-manager - - - - diff --git a/pom.xml b/pom.xml index 0fc02bae1..217a266c9 100644 --- a/pom.xml +++ b/pom.xml @@ -84,8 +84,8 @@ 0.6.1 3.8.1 3.0.8 - 2.10.0 - 1.6.0 + 3.3.0 + 2.3.0 5.7.2 2.6.2 3.15.8