From a3424a6875e9732d9b2c4256f398386aef14afcb Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 19 Aug 2024 19:03:13 +0530 Subject: [PATCH] Re-factor some of the remote store settings Signed-off-by: Sachin Kale --- .../index/remote/RemoteStoreUtils.java | 22 +++++++++++++++++++ .../store/RemoteSegmentStoreDirectory.java | 6 ++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index ea6b1b13278b4..6569e7c699c2b 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -21,8 +21,10 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.node.remotestore.RemoteStoreNodeService; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -511,4 +513,24 @@ public static List filterOutMetadataFilesBasedOnAge( } return metadataFilesWithMinAge; } + + /** + * Determines if the pinned timestamp state is stale based on the provided remote store settings. + * + * This method checks if the last successful fetch timestamp is older than a calculated stale buffer time. + * The stale buffer time is computed using the pinned timestamps scheduler interval and lookback interval + * from the remote store settings. + * + * @return true if the pinned timestamp state is considered stale, false otherwise. + * + * @throws NullPointerException if remoteStoreSettings is null. + * @throws IllegalStateException if unable to retrieve the pinned timestamps. + */ + public static boolean isPinnedTimestampStateStale() { + long lastSuccessfulFetchTimestamp = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1(); + long staleBufferInMillis = (RemoteStoreSettings.getPinnedTimestampsSchedulerInterval().millis() * 3) + RemoteStoreSettings + .getPinnedTimestampsLookbackInterval() + .millis(); + return lastSuccessfulFetchTimestamp < (System.currentTimeMillis() - staleBufferInMillis); + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index f267b7ea8f5b6..6b31f74a3caf2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -40,6 +40,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.threadpool.ThreadPool; @@ -830,7 +831,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException } // Check last fetch status of pinned timestamps. If stale, return. - if (RemoteStorePinnedTimestampService.isPinnedTimestampStateStale()) { + if (RemoteStoreUtils.isPinnedTimestampStateStale()) { logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); return; } @@ -861,8 +862,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException // Along with last N files, we need to keep files since last successful run of scheduler long lastSuccessfulFetchOfPinnedTimestamps = pinnedTimestampsState.v1(); - long minimumAgeInMillis = lastSuccessfulFetchOfPinnedTimestamps + RemoteStorePinnedTimestampService - .getPinnedTimestampsLookbackInterval() + long minimumAgeInMillis = lastSuccessfulFetchOfPinnedTimestamps + RemoteStoreSettings.getPinnedTimestampsLookbackInterval() .getMillis(); metadataFilesEligibleToDelete = RemoteStoreUtils.filterOutMetadataFilesBasedOnAge( metadataFilesEligibleToDelete,