chore: check tsm overlap when chunk merge

Subsegment · Nov 8, 2024 · 88f97e6 · 88f97e6
1 parent 9b2b4a6
commit 88f97e6
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 10 deletions.
diff --git a/tskv/src/reader/iterator.rs b/tskv/src/reader/iterator.rs
@@ -22,7 +22,7 @@ use snafu::ResultExt;
 use tokio::runtime::Runtime;
 use tokio_util::sync::CancellationToken;
 use trace::span_ext::SpanExt;
-use trace::{debug, Span, SpanContext};
+use trace::{debug, error, Span, SpanContext};
 
 use super::display::DisplayableBatchReader;
 use super::memcache_reader::MemCacheReader;
@@ -430,7 +430,17 @@ impl SeriesGroupBatchReaderFactory {
         };
 
         let mut chunk_readers = Vec::new();
+        let mut only_tsm = true;
         for data_reference in chunks.into_iter() {
+            match data_reference {
+                DataReference::Memcache(_, _, _) => {
+                    only_tsm = false;
+                }
+                DataReference::Chunk(_, _, ref file) if file.is_delta() => {
+                    only_tsm = false;
+                }
+                _ => {}
+            }
             let chunk_reader = self.build_chunk_reader(
                 data_reference,
                 batch_size,
@@ -443,7 +453,9 @@ impl SeriesGroupBatchReaderFactory {
                 chunk_readers.push(chunk_reader);
             }
         }
-
+        if only_tsm && chunk_readers.len() > 1 {
+            error!("Tsm files have data overlap, please check the compaction process");
+        }
         Ok(chunk_readers)
     }
 

diff --git a/tskv/src/reader/mod.rs b/tskv/src/reader/mod.rs
@@ -308,14 +308,7 @@ impl PartialOrd<Self> for DataReference {
 
 impl Ord for DataReference {
     fn cmp(&self, other: &Self) -> Ordering {
-        match (self, other) {
-            (DataReference::Chunk(_, _, _), DataReference::Memcache(_, _, _)) => Ordering::Less,
-            (DataReference::Memcache(_, _, _), DataReference::Chunk(_, _, _)) => Ordering::Greater,
-            (DataReference::Chunk(_, _, f1), DataReference::Chunk(_, _, f2)) => {
-                f1.file_id().cmp(&f2.file_id())
-            }
-            (DataReference::Memcache(_, _, c1), DataReference::Memcache(_, _, c2)) => c1.cmp(c2),
-        }
+        self.file_id().cmp(&other.file_id())
     }
 }
 
@@ -326,6 +319,13 @@ impl DataReference {
             DataReference::Memcache(_, trs, ..) => trs.max_time_range(),
         }
     }
+
+    pub fn file_id(&self) -> ColumnFileId {
+        match self {
+            DataReference::Chunk(_, _, cf) => cf.file_id(),
+            DataReference::Memcache(_, _, cf_id) => *cf_id,
+        }
+    }
 }
 
 impl TimeRangeProvider for DataReference {