feat(mito): checkpoint for mito2 (#2142)

* basic impl Signed-off-by: Ruihang Xia <[email protected]> * adjust dir structure Signed-off-by: Ruihang Xia <[email protected]> * add tests Signed-off-by: Ruihang Xia <[email protected]> * fix styles Signed-off-by: Ruihang Xia <[email protected]> * fix typo Signed-off-by: Ruihang Xia <[email protected]> * sort result Signed-off-by: Ruihang Xia <[email protected]> * downgrade log level Signed-off-by: Ruihang Xia <[email protected]> * apply CR sugg. Signed-off-by: Ruihang Xia <[email protected]> * add region id to log Signed-off-by: Ruihang Xia <[email protected]> --------- Signed-off-by: Ruihang Xia <[email protected]>
GreptimeTeam · Aug 13, 2023 · 6d64e1c · 6d64e1c
1 parent e6090a8
commit 6d64e1c
Show file tree

Hide file tree

Showing 16 changed files with 457 additions and 223 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml
@@ -27,6 +27,7 @@ orc-rust = "0.2"
 paste = "1.0"
 regex = "1.7"
 snafu.workspace = true
+strum = { version = "0.21", features = ["derive"] }
 tokio-util.workspace = true
 tokio.workspace = true
 url = "2.3"

diff --git a/src/common/datasource/src/compression.rs b/src/common/datasource/src/compression.rs
@@ -20,11 +20,12 @@ use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdD
 use async_compression::tokio::write;
 use bytes::Bytes;
 use futures::Stream;
+use strum::EnumIter;
 use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
 use tokio_util::io::{ReaderStream, StreamReader};
 
 use crate::error::{self, Error, Result};
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)]
 pub enum CompressionType {
     /// Gzip-ed file
     Gzip,

diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml
@@ -46,6 +46,7 @@ serde_json = "1.0"
 snafu.workspace = true
 storage = { workspace = true }
 store-api = { workspace = true }
+strum = "0.21"
 table = { workspace = true }
 tokio.workspace = true
 uuid.workspace = true

diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
@@ -37,7 +37,7 @@ pub struct MitoConfig {
     // Manifest configs:
     /// Number of meta action updated to trigger a new checkpoint
     /// for the manifest (default 10).
-    pub manifest_checkpoint_interval: u64,
+    pub manifest_checkpoint_distance: u64,
     /// Manifest compression type (default uncompressed).
     pub manifest_compress_type: CompressionType,
 }
@@ -48,7 +48,7 @@ impl Default for MitoConfig {
             num_workers: DEFAULT_NUM_WORKERS,
             worker_channel_size: 128,
             worker_request_batch_size: 64,
-            manifest_checkpoint_interval: 10,
+            manifest_checkpoint_distance: 10,
             manifest_compress_type: CompressionType::Uncompressed,
         }
     }

diff --git a/src/mito2/src/manifest.rs b/src/mito2/src/manifest.rs
@@ -15,9 +15,8 @@
 //! manifest storage
 
 pub mod action;
-pub mod gc_task;
-pub mod helper;
 #[allow(unused_variables)]
 pub mod manager;
-pub mod options;
 pub mod storage;
+#[cfg(test)]
+mod tests;
diff --git a/src/mito2/src/manifest/action.rs b/src/mito2/src/manifest/action.rs
@@ -12,18 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Defines [RegionMetaAction] related structs and [RegionCheckpoint].
+
 use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
-use storage::metadata::VersionNumber;
-use storage::sst::{FileId, FileMeta};
 use store_api::manifest::action::{ProtocolAction, ProtocolVersion};
 use store_api::manifest::ManifestVersion;
 use store_api::storage::{RegionId, SequenceNumber};
 
 use crate::error::{RegionMetadataNotFoundSnafu, Result, SerdeJsonSnafu, Utf8Snafu};
 use crate::metadata::RegionMetadataRef;
+use crate::sst::file::{FileId, FileMeta};
 
 /// Actions that can be applied to region manifest.
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
@@ -46,7 +47,6 @@ pub struct RegionChange {
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 pub struct RegionEdit {
-    pub region_version: VersionNumber,
     pub files_to_add: Vec<FileMeta>,
     pub files_to_remove: Vec<FileMeta>,
     pub compaction_time_window: Option<i64>,
@@ -123,8 +123,6 @@ impl RegionManifestBuilder {
 // The checkpoint of region manifest, generated by checkpointer.
 #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
 pub struct RegionCheckpoint {
-    /// The snasphot protocol
-    pub protocol: ProtocolAction,
     /// The last manifest version that this checkpoint compacts(inclusive).
     pub last_version: ManifestVersion,
     // The number of manifest actions that this checkpoint compacts.
@@ -134,21 +132,20 @@ pub struct RegionCheckpoint {
 }
 
 impl RegionCheckpoint {
-    pub fn set_protocol(&mut self, action: ProtocolAction) {
-        self.protocol = action;
-    }
-
     pub fn last_version(&self) -> ManifestVersion {
         self.last_version
     }
 
     pub fn encode(&self) -> Result<Vec<u8>> {
-        todo!()
+        let json = serde_json::to_string(&self).context(SerdeJsonSnafu)?;
+
+        Ok(json.into_bytes())
     }
 
-    pub fn decode(bs: &[u8]) -> Result<Self> {
-        // helper::decode_checkpoint(bs, reader_version)
-        todo!()
+    pub fn decode(bytes: &[u8]) -> Result<Self> {
+        let data = std::str::from_utf8(bytes).context(Utf8Snafu)?;
+
+        serde_json::from_str(data).context(SerdeJsonSnafu)
     }
 }
 
@@ -207,7 +204,6 @@ impl RegionMetaActionIter {
 
 #[cfg(test)]
 mod tests {
-    use storage::sst::FileId;
 
     use super::*;
 
@@ -238,7 +234,7 @@ mod tests {
         FileMeta {
             region_id: 0.into(),
             file_id: FileId::random(),
-            time_range: None,
+            time_range: (0.into(), 10000.into()),
             level: 0,
             file_size: 1024,
         }

diff --git a/src/mito2/src/manifest/helper.rs b/src/mito2/src/manifest/helper.rs