[feat](backup) GetSnapshot returns snapshot expiration time (apache#4…

…3731) ### What problem does this PR solve? Related PR: selectdb/ccr-syncer#229 Return the expiration time of the snapshot
924060929 · Nov 13, 2024 · 5d99f2b · 5d99f2b
1 parent 7988fed
commit 5d99f2b
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 6 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -1027,12 +1027,18 @@ public synchronized Snapshot getSnapshot() {
             return null;
         }
 
+        // Avoid loading expired meta.
+        long expiredAt = createTime + timeoutMs;
+        if (System.currentTimeMillis() >= expiredAt) {
+            return new Snapshot(label, new byte[0], new byte[0], expiredAt);
+        }
+
         try {
             File metaInfoFile = new File(localMetaInfoFilePath);
             File jobInfoFile = new File(localJobInfoFilePath);
             byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath());
             byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath());
-            return new Snapshot(label, metaInfoBytes, jobInfoBytes);
+            return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt);
         } catch (IOException e) {
             LOG.warn("failed to load meta info and job info file, meta info file {}, job info file {}: ",
                     localMetaInfoFilePath, localJobInfoFilePath, e);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
@@ -31,16 +31,19 @@ public class Snapshot {
     @SerializedName(value = "jobInfo")
     private byte[] jobInfo = null;
 
+    @SerializedName(value = "expired_at")
+    private long expiredAt = 0;
+
     public Snapshot() {
     }
 
-    public Snapshot(String label, byte[] meta, byte[] jobInfo) {
+    public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt) {
         this.label = label;
         this.meta = meta;
         this.jobInfo = jobInfo;
+        this.expiredAt = expiredAt;
     }
 
-
     public byte[] getMeta() {
         return meta;
     }
@@ -49,17 +52,25 @@ public byte[] getJobInfo() {
         return jobInfo;
     }
 
+    public long getExpiredAt() {
+        return expiredAt;
+    }
+
+    public boolean isExpired() {
+        return System.currentTimeMillis() > expiredAt;
+    }
+
     public String toJson() {
         return GsonUtils.GSON.toJson(this);
     }
 
     @Override
     public String toString() {
-        // return toJson();
         return "Snapshot{"
                 + "label='" + label + '\''
                 + ", meta=" + meta
                 + ", jobInfo=" + jobInfo
+                + ", expiredAt=" + expiredAt
                 + '}';
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -2932,12 +2932,16 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c
         if (snapshot == null) {
             result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_NOT_EXIST);
             result.getStatus().addToErrorMsgs(String.format("snapshot %s not exist", label));
+        } else if (snapshot.isExpired()) {
+            result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_EXPIRED);
+            result.getStatus().addToErrorMsgs(String.format("snapshot %s is expired", label));
         } else {
             byte[] meta = snapshot.getMeta();
             byte[] jobInfo = snapshot.getJobInfo();
+            long expiredAt = snapshot.getExpiredAt();
 
-            LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}",
-                    label, meta.length, jobInfo.length);
+            LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}, expired at: {}",
+                    label, meta.length, jobInfo.length, expiredAt);
             if (request.isEnableCompress()) {
                 meta = GZIPUtils.compress(meta);
                 jobInfo = GZIPUtils.compress(jobInfo);
@@ -2949,6 +2953,7 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c
             }
             result.setMeta(meta);
             result.setJobInfo(jobInfo);
+            result.setExpiredAt(expiredAt);
         }
 
         return result;

diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
@@ -1362,6 +1362,7 @@ struct TGetSnapshotResult {
     3: optional binary job_info
     4: optional Types.TNetworkAddress master_address
     5: optional bool compressed;
+    6: optional i64 expiredAt;  // in millis
 }
 
 struct TTableRef {

diff --git a/gensrc/thrift/Status.thrift b/gensrc/thrift/Status.thrift
@@ -106,6 +106,8 @@ enum TStatusCode {
 
     OBTAIN_LOCK_FAILED = 74,
 
+    SNAPSHOT_EXPIRED = 75,
+
     // used for cloud
     DELETE_BITMAP_LOCK_ERROR = 100,
     // Not be larger than 200, see status.h

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -486,6 +486,8 @@ class Syncer {
                             logger.error("TGetSnapshotResult meta is unset.")
                         } else if (!result.isSetJobInfo()) {
                             logger.error("TGetSnapshotResult job info is unset.")
+                        } else if (!result.isSetExpiredAt()) {
+                            logger.error("TGetSnapshotResult expiredAt is unset.")
                         } else {
                             isCheckedOK = true
                         }