Halt TabletServer on minc failure and no TabletServer lock (apache#5169)

Related to apache#5137
dlmarion · Jan 29, 2025 · 7679f0c · 7679f0c
1 parent 712e50e
commit 7679f0c
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 7 deletions.
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
@@ -29,8 +29,10 @@
 
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
 import org.apache.accumulo.core.manager.state.tables.TableState;
 import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.util.Halt;
 import org.apache.accumulo.core.util.LocalityGroupUtil;
 import org.apache.accumulo.server.compaction.CompactionStats;
 import org.apache.accumulo.server.compaction.FileCompactor;
@@ -93,11 +95,19 @@ public CompactionStats call() {
     try {
       do {
         try {
-          CompactionStats ret = super.call();
-
-          // log.debug(String.format("MinC %,d recs in | %,d recs out | %,d recs/sec | %6.3f secs |
-          // %,d bytes ",map.size(), entriesCompacted,
-          // (int)(map.size()/((t2 - t1)/1000.0)), (t2 - t1)/1000.0, estimatedSizeInBytes()));
+          CompactionStats ret = null;
+          try {
+            ret = super.call();
+          } catch (Exception e) {
+            final ServiceLock tserverLock = tabletServer.getLock();
+            if (tserverLock == null || !tserverLock.verifyLockAtSource()) {
+              log.error("Minor compaction of {} has failed and TabletServer lock does not exist."
+                  + " Halting...", getExtent(), e);
+              Halt.halt("TabletServer lock does not exist", -1);
+            } else {
+              throw e;
+            }
+          }
 
           if (reportedProblem) {
             ProblemReports.getInstance(tabletServer.getContext())

diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -65,6 +65,7 @@
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.dataImpl.KeyExtent;
 import org.apache.accumulo.core.dataImpl.thrift.MapFileInfo;
+import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
 import org.apache.accumulo.core.file.FileOperations;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator;
@@ -91,6 +92,7 @@
 import org.apache.accumulo.core.tabletserver.log.LogEntry;
 import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.core.trace.TraceUtil;
+import org.apache.accumulo.core.util.Halt;
 import org.apache.accumulo.core.util.Pair;
 import org.apache.accumulo.core.volume.Volume;
 import org.apache.accumulo.server.ServerContext;
@@ -508,8 +510,15 @@ DataFileValue minorCompact(InMemoryMap memTable, TabletFile tmpDatafile, TabletF
             flushId);
         storedFile.ifPresent(stf -> compactable.filesAdded(true, List.of(stf)));
       } catch (Exception e) {
-        TraceUtil.setException(span2, e, true);
-        throw e;
+        final ServiceLock tserverLock = tabletServer.getLock();
+        if (tserverLock == null || !tserverLock.verifyLockAtSource()) {
+          log.error("Minor compaction of {} has failed and TabletServer lock does not exist."
+              + " Halting...", getExtent(), e);
+          Halt.halt("TabletServer lock does not exist", -1);
+        } else {
+          TraceUtil.setException(span2, e, true);
+          throw e;
+        }
       } finally {
         span2.end();
       }