diff --git a/direct/io-bigtable/pom.xml b/direct/io-bigtable/pom.xml
index 0239c339b..4198dfd04 100644
--- a/direct/io-bigtable/pom.xml
+++ b/direct/io-bigtable/pom.xml
@@ -33,8 +33,8 @@
cz.o2.proxima.bigtable.shaded
- 1.12.0
- 1.21.0
+ 1.19.0
+ 1.33.1
@@ -219,7 +219,7 @@
com.google.cloud.bigtable
- bigtable-hbase-1.x
+ bigtable-hbase-2.x
${bigtable.version}
diff --git a/direct/io-hbase-bindings/pom.xml b/direct/io-hbase-bindings/pom.xml
index a5b5e43e3..1b23ef025 100644
--- a/direct/io-hbase-bindings/pom.xml
+++ b/direct/io-hbase-bindings/pom.xml
@@ -179,6 +179,12 @@
${guava.version}
+
+ org.apache.hadoop
+ hadoop-auth
+ ${hadoop.version}
+
+
org.apache.hadoop
hadoop-common
@@ -206,6 +212,34 @@
+
+ org.apache.hadoop
+ hadoop-minicluster
+ ${hadoop.version}
+ test
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop.version}
+ test
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+ ${hadoop.version}
+ test
+
+
+
+ org.apache.hadoop
+ hadoop-client
+ ${hadoop.version}
+ test
+
+
org.apache.hadoop
hadoop-hdfs
diff --git a/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java b/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java
index b64a22ba3..d5b42fc52 100644
--- a/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java
+++ b/direct/io-hbase-bindings/src/main/java/cz/o2/proxima/direct/hbase/HBaseWriter.java
@@ -32,7 +32,6 @@
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -45,24 +44,17 @@
class HBaseWriter extends HBaseClientWrapper implements OnlineAttributeWriter {
private static final String DEL_BATCH_SIZE_CONF = "del-batch-size";
- private static final String FLUSH_COMMITS_CFG = "flush-commits";
private final int batchSize;
private final Map cfg;
private final InternalSerializer serializer;
- private boolean flushCommits;
-
HBaseWriter(URI uri, Configuration conf, Map cfg) {
super(uri, conf);
batchSize =
Optional.ofNullable(cfg.get(DEL_BATCH_SIZE_CONF))
.map(o -> Integer.valueOf(o.toString()))
.orElse(1000);
- flushCommits =
- Optional.ofNullable(cfg.get(FLUSH_COMMITS_CFG))
- .map(o -> Boolean.valueOf(o.toString()))
- .orElse(true);
serializer = HBaseDataAccessor.instantiateSerializer(uri);
this.cfg = cfg;
}
@@ -89,9 +81,6 @@ public void write(StreamElement data, CommitCallback statusCallback) {
Put put = serializer.toPut(family, key, data);
this.client.put(put);
}
- if (flushCommits) {
- ((HTable) this.client).flushCommits();
- }
statusCallback.commit(true, null);
} catch (Exception ex) {
log.error("Failed to write {}", data, ex);
@@ -148,12 +137,4 @@ private void deletePrefix(byte[] key, byte[] family, String prefix, long stamp)
stamp);
}
}
-
- @Override
- void ensureClient() {
- super.ensureClient();
- if (!(client instanceof HTable)) {
- flushCommits = false;
- }
- }
}
diff --git a/direct/pom.xml b/direct/pom.xml
index 3556c066b..662e3a99f 100644
--- a/direct/pom.xml
+++ b/direct/pom.xml
@@ -66,63 +66,5 @@
-
-
- jdk8
-
- 1.8
-
-
- compiler
- core
- core-testing
- ingest-client
- ingest-server
- io-bigtable
- io-blob
- io-bulkfs
- io-bulkfs-parquet
- io-cassandra
- io-gcloud-storage
- io-hadoop
- io-hbase
- io-hbase-bindings
- io-hdfs
- io-http
- io-kafka
- io-pubsub
- io-s3
- rpc
- transaction-manager
-
-
-
-
- jdk11
-
- 11
-
-
- compiler
- core
- core-testing
- ingest-client
- ingest-server
- io-bigtable
- io-blob
- io-bulkfs
- io-bulkfs-parquet
- io-cassandra
- io-gcloud-storage
- io-http
- io-kafka
- io-pubsub
- io-s3
- rpc
- transaction-manager
-
-
-
-
diff --git a/pom.xml b/pom.xml
index 0fc02bae1..217a266c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,8 +84,8 @@
0.6.1
3.8.1
3.0.8
- 2.10.0
- 1.6.0
+ 3.3.0
+ 2.3.0
5.7.2
2.6.2
3.15.8