Remove ZK compaction and compaction cancellation ids (apache#3963)

Removed compact columns during upgrade and from code

Fixes apache#3584

core/src/main/java/org/apache/accumulo/core/Constants.java

@@ -44,11 +44,6 @@ public class Constants {
   public static final String ZTABLE_STATE = "/state";
   public static final String ZTABLE_FLUSH_ID = "/flush-id";
 
-  // ELASTICITY_TODO delete from code and remove from ZK in upgrade
-  public static final String ZTABLE_COMPACT_ID = "/compact-id";
-
-  // ELASTICITY_TODO delete from code and remove from ZK in upgrade
-  public static final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
   public static final String ZTABLE_NAMESPACE = "/namespace";
 
   public static final String ZNAMESPACES = "/namespaces";

core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java

@@ -357,8 +357,6 @@ interface TabletUpdates<T> {
 
     T deleteScan(StoredTabletFile path);
 
-    T putCompactionId(long compactionId);
-
     T putFlushId(long flushId);
 
     T putLocation(Location location);

core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java

@@ -242,9 +242,6 @@ public static void validateDirCol(String dirName) {
       public static final String FLUSH_QUAL = "flush";
       public static final ColumnFQ FLUSH_COLUMN = new ColumnFQ(NAME, new Text(FLUSH_QUAL));
 
-      // ELASTICITY_TODO remove this from code and remove it from metadata in upgrade
-      public static final String COMPACT_QUAL = "compact";
-      public static final ColumnFQ COMPACT_COLUMN = new ColumnFQ(NAME, new Text(COMPACT_QUAL));
       /**
        * Holds lock IDs to enable a sanity check to ensure that the TServer writing to the metadata
        * tablet is not dead

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java

@@ -20,7 +20,6 @@
 
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_QUAL;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.REQUESTED_QUAL;
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_QUAL;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_QUAL;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_QUAL;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_QUAL;
@@ -116,7 +115,6 @@ public class TabletMetadata {
   private SortedMap<Key,Value> keyValues;
   private OptionalLong flush = OptionalLong.empty();
   private List<LogEntry> logs;
-  private OptionalLong compact = OptionalLong.empty();
   private Double splitRatio = null;
   private Map<ExternalCompactionId,ExternalCompactionMetadata> extCompactions;
   private boolean merged;
@@ -147,7 +145,6 @@ public enum ColumnType {
     CLONED,
     FLUSH_ID,
     LOGS,
-    COMPACT_ID,
     SPLIT_RATIO,
     SUSPEND,
     ECOMP,
@@ -372,11 +369,6 @@ public OptionalLong getFlushId() {
     return flush;
   }
 
-  public OptionalLong getCompactId() {
-    ensureFetched(ColumnType.COMPACT_ID);
-    return compact;
-  }
-
   public Double getSplitRatio() {
     ensureFetched(ColumnType.SPLIT_RATIO);
     return splitRatio;
@@ -411,9 +403,8 @@ public String toString() {
         .append("fetchedCols", fetchedCols).append("extent", extent).append("last", last)
         .append("suspend", suspend).append("dirName", dirName).append("time", time)
         .append("cloned", cloned).append("flush", flush).append("logs", logs)
-        .append("compact", compact).append("splitRatio", splitRatio)
-        .append("extCompactions", extCompactions).append("goal", goal)
-        .append("onDemandHostingRequested", onDemandHostingRequested)
+        .append("splitRatio", splitRatio).append("extCompactions", extCompactions)
+        .append("goal", goal).append("onDemandHostingRequested", onDemandHostingRequested)
         .append("operationId", operationId).append("selectedFiles", selectedFiles)
         .append("futureAndCurrentLocationSet", futureAndCurrentLocationSet).toString();
   }
@@ -514,9 +505,6 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
             case FLUSH_QUAL:
               te.flush = OptionalLong.of(Long.parseLong(val));
               break;
-            case COMPACT_QUAL:
-              te.compact = OptionalLong.of(Long.parseLong(val));
-              break;
             case OPID_QUAL:
               te.setOperationIdOnce(val, suppressLocationError);
               break;

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java

@@ -19,7 +19,6 @@
 package org.apache.accumulo.core.metadata.schema;
 
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
-import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACT_ID;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
@@ -115,13 +114,6 @@ public TabletMetadataBuilder deleteScan(StoredTabletFile path) {
     throw new UnsupportedOperationException();
   }
 
-  @Override
-  public TabletMetadataBuilder putCompactionId(long compactionId) {
-    fetched.add(COMPACT_ID);
-    internalBuilder.putCompactionId(compactionId);
-    return this;
-  }
-
   @Override
   public TabletMetadataBuilder putFlushId(long flushId) {
     fetched.add(FLUSH_ID);

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java

@@ -128,13 +128,6 @@ public T deleteScan(StoredTabletFile path) {
     return getThis();
   }
 
-  @Override
-  public T putCompactionId(long compactionId) {
-    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
-    ServerColumnFamily.COMPACT_COLUMN.put(mutation, new Value(Long.toString(compactionId)));
-    return getThis();
-  }
-
   @Override
   public T putFlushId(long flushId) {
     Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java

@@ -22,7 +22,6 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
@@ -301,9 +300,6 @@ public Options fetch(ColumnType... colsToFetch) {
           case CLONED:
             families.add(ClonedColumnFamily.NAME);
             break;
-          case COMPACT_ID:
-            qualifiers.add(COMPACT_COLUMN);
-            break;
           case DIR:
             qualifiers.add(DIRECTORY_COLUMN);
             break;

core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java

@@ -23,7 +23,6 @@
 import static org.apache.accumulo.core.metadata.StoredTabletFile.serialize;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_VALUE;
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
@@ -91,7 +90,6 @@ public void testAllColumns() {
 
     Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
 
-    COMPACT_COLUMN.put(mutation, new Value("5"));
     DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
     FLUSH_COLUMN.put(mutation, new Value("6"));
     TIME_COLUMN.put(mutation, new Value("M123456789"));
@@ -134,7 +132,6 @@ public void testAllColumns() {
         EnumSet.allOf(ColumnType.class), true, false);
 
     assertEquals("OK", tm.getCloned());
-    assertEquals(5L, tm.getCompactId().getAsLong());
     assertEquals("t-0001757", tm.getDirName());
     assertEquals(extent.endRow(), tm.getEndRow());
     assertEquals(extent, tm.getExtent());
@@ -351,14 +348,12 @@ public void testBuilder() {
 
     TabletMetadata tm = TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.NEVER)
         .putLocation(Location.future(ser1)).putFile(sf1, dfv1).putFile(sf2, dfv2)
-        .putCompactionId(23).putBulkFile(rf1, 25).putBulkFile(rf2, 35).putFlushId(27)
-        .putDirName("dir1").putScan(sf3).putScan(sf4).putCompacted(17).putCompacted(23)
-        .build(ECOMP, HOSTING_REQUESTED, MERGED);
+        .putBulkFile(rf1, 25).putBulkFile(rf2, 35).putFlushId(27).putDirName("dir1").putScan(sf3)
+        .putScan(sf4).putCompacted(17).putCompacted(23).build(ECOMP, HOSTING_REQUESTED, MERGED);
 
     assertEquals(extent, tm.getExtent());
     assertEquals(TabletHostingGoal.NEVER, tm.getHostingGoal());
     assertEquals(Location.future(ser1), tm.getLocation());
-    assertEquals(23L, tm.getCompactId().orElse(-1));
     assertEquals(27L, tm.getFlushId().orElse(-1));
     assertEquals(Map.of(sf1, dfv1, sf2, dfv2), tm.getFilesMap());
     assertEquals(Map.of(rf1.insert(), 25L, rf2.insert(), 35L), tm.getLoaded());
@@ -380,7 +375,6 @@ public void testBuilder() {
     assertNull(tm2.getLocation());
     assertThrows(IllegalStateException.class, tm2::getFiles);
     assertThrows(IllegalStateException.class, tm2::getHostingGoal);
-    assertThrows(IllegalStateException.class, tm2::getCompactId);
     assertThrows(IllegalStateException.class, tm2::getFlushId);
     assertThrows(IllegalStateException.class, tm2::getFiles);
     assertThrows(IllegalStateException.class, tm2::getLogs);

server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java

@@ -89,7 +89,6 @@ public class MetadataConstraints implements Constraint {
           ServerColumnFamily.TIME_COLUMN,
           ServerColumnFamily.LOCK_COLUMN,
           ServerColumnFamily.FLUSH_COLUMN,
-          ServerColumnFamily.COMPACT_COLUMN,
           ServerColumnFamily.OPID_COLUMN,
           HostingColumnFamily.GOAL_COLUMN,
           HostingColumnFamily.REQUESTED_COLUMN,

server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java

@@ -20,9 +20,7 @@
 package org.apache.accumulo.server.metadata;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_COLUMN;
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.SELECTED_COLUMN;
@@ -49,6 +47,7 @@
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
@@ -163,15 +162,6 @@ private void requireSameSingle(TabletMetadata tabletMetadata, ColumnType type) {
     switch (type) {
       case PREV_ROW:
         throw new IllegalStateException("PREV_ROW already set from Extent");
-      case COMPACT_ID: {
-        Condition c =
-            new Condition(COMPACT_COLUMN.getColumnFamily(), COMPACT_COLUMN.getColumnQualifier());
-        if (tabletMetadata.getCompactId().isPresent()) {
-          c = c.setValue(Long.toString(tabletMetadata.getCompactId().getAsLong()));
-        }
-        mutation.addCondition(c);
-      }
-        break;
       case LOGS: {
         Condition c = SetEqualityIterator.createCondition(new HashSet<>(tabletMetadata.getLogs()),
             logEntry -> logEntry.getColumnQualifier().toString().getBytes(UTF_8),

server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java

@@ -104,8 +104,6 @@ public static void prepareNewTableState(ZooReaderWriter zoo, PropStore propStore
     zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME, tableName.getBytes(UTF_8),
         existsPolicy);
     zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE, existsPolicy);
-    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE, existsPolicy);
-    zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID, ZERO_BYTE, existsPolicy);
     zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE, state.name().getBytes(UTF_8),
         existsPolicy);
     var propKey = TablePropKey.of(instanceId, tableId);

server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java

@@ -61,7 +61,7 @@ public class ManagerMetadataUtil {
   public static void addNewTablet(ServerContext context, KeyExtent extent, String dirName,
       TServerInstance tServerInstance, Map<StoredTabletFile,DataFileValue> datafileSizes,
       Map<Long,? extends Collection<ReferencedTabletFile>> bulkLoadedFiles, MetadataTime time,
-      long lastFlushID, long lastCompactID, ServiceLock zooLock) {
+      long lastFlushID, ServiceLock zooLock) {
 
     // ELASTICITY_TODO intentionally not using conditional mutations for this code because its only
     // called when tablets split. Tablet splitting will drastically change, so there is no need to
@@ -77,10 +77,6 @@ public static void addNewTablet(ServerContext context, KeyExtent extent, String
       tablet.putFlushId(lastFlushID);
     }
 
-    if (lastCompactID > 0) {
-      tablet.putCompactionId(lastCompactID);
-    }
-
     if (tServerInstance != null) {
       tablet.putLocation(Location.current(tServerInstance));
       tablet.deleteLocation(Location.future(tServerInstance));

server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java

@@ -18,7 +18,6 @@
  */
 package org.apache.accumulo.server.manager.state;
 
-import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
 import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
@@ -73,7 +72,6 @@ private SortedMap<Key,Value> createMetadataEntryKV(KeyExtent extent) {
 
     Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
 
-    COMPACT_COLUMN.put(mutation, new Value("5"));
     DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
     FLUSH_COLUMN.put(mutation, new Value("6"));
     TIME_COLUMN.put(mutation, new Value("M123456789"));