From efcb97b82292b767654b6b47793ffa621aa719b6 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Tue, 17 Dec 2024 16:34:17 +0800 Subject: [PATCH 1/8] support --- Cargo.lock | 26 ++++++++ src/common/Cargo.toml | 1 + src/common/src/types/datetime.rs | 62 ++++++++++--------- .../compaction_group/hummock_version_ext.rs | 5 +- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b242d74d2e06f..9e2ddd20dc3f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6758,6 +6758,31 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "jiff" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db69f08d4fb10524cacdb074c10b296299d71274ddbc830a8ee65666867002e9" +dependencies = [ + "jiff-tzdb-platform", + "windows-sys 0.59.0", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91335e575850c5c4c673b9bd467b0e025f164ca59d0564f69d0c2ee0ffad4653" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9835f0060a626fe59f160437bc725491a6af23133ea906500027d1bd2f8f4329" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -10741,6 +10766,7 @@ dependencies = [ "hytra", "itertools 0.13.0", "itoa", + "jiff", "jsonbb", "libc", "lru 0.7.6", diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 0aa9498a7acc2..5c6f526d7b506 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -56,6 +56,7 @@ humantime = "2.1" hytra = { workspace = true } itertools = { workspace = true } itoa = "1.0" +jiff = "0.1.15" jsonbb = { workspace = true } lru = { workspace = true } memcomparable = { version = "0.2", features = ["decimal"] } diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs index 1b8d004c68b2b..b412e3c15f637 100644 --- a/src/common/src/types/datetime.rs +++ b/src/common/src/types/datetime.rs @@ -168,29 +168,18 @@ impl FromStr for Timestamp { type Err = InvalidParamsError; fn from_str(s: &str) -> Result { - if let Ok(res) = speedate::DateTime::parse_str_rfc3339(s) { - if res.time.tz_offset.is_some() { - return Err(ErrorKind::ParseTimestamp.into()); - } - Ok(Date::from_ymd_uncheck( - res.date.year as i32, - res.date.month as u32, - res.date.day as u32, - ) - .and_hms_micro_uncheck( - res.time.hour as u32, - res.time.minute as u32, - res.time.second as u32, - res.time.microsecond, - )) - } else { - let res = - speedate::Date::parse_str_rfc3339(s).map_err(|_| ErrorKind::ParseTimestamp)?; - Ok( - Date::from_ymd_uncheck(res.year as i32, res.month as u32, res.day as u32) - .and_hms_micro_uncheck(0, 0, 0, 0), - ) - } + let dt = s + .parse::() + .map_err(|_| ErrorKind::ParseTimestamp)?; + Ok( + Date::from_ymd_uncheck(dt.year() as i32, dt.month() as u32, dt.day() as u32) + .and_hms_nano_uncheck( + dt.hour() as u32, + dt.minute() as u32, + dt.second() as u32, + dt.subsec_nanosecond() as u32, + ), + ) } } @@ -422,6 +411,13 @@ impl Date { .and_time(Time::from_hms_micro_uncheck(hour, min, sec, micro).0), ) } + + pub fn and_hms_nano_uncheck(self, hour: u32, min: u32, sec: u32, nano: u32) -> Timestamp { + Timestamp::new( + self.0 + .and_time(Time::from_hms_nano_uncheck(hour, min, sec, nano).0), + ) + } } impl Time { @@ -495,18 +491,24 @@ impl Timestamp { } pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult { - let micros = cur + let secs = cur .read_i64::() - .context("failed to read i64 from Timestamp buffer")?; + .context("failed to read i64 from Time buffer")?; + let nsecs = cur + .read_u32::() + .context("failed to read u32 from Time buffer")?; - Ok(Timestamp::with_micros(micros)?) + Ok(Timestamp::with_secs_nsecs(secs, nsecs)?) } - /// Although `Timestamp` takes 12 bytes, we drop 4 bytes in protobuf encoding. pub fn to_protobuf(self, output: &mut T) -> ArrayResult { - output - .write(&(self.0.and_utc().timestamp_micros()).to_be_bytes()) - .map_err(Into::into) + let timestamp_size = output + .write(&(self.0.and_utc().timestamp()).to_be_bytes()) + .map_err(Into::::into)?; + let timestamp_subsec_nanos_size = output + .write(&(self.0.and_utc().timestamp_subsec_nanos()).to_be_bytes()) + .map_err(Into::::into)?; + Ok(timestamp_subsec_nanos_size + timestamp_size) } pub fn get_timestamp_nanos(&self) -> i64 { diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index bcf99e18bfe7a..fc8514eac8119 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -524,7 +524,7 @@ impl HummockVersion { .clone_from(&group_construct.table_ids); self.levels.insert(*compaction_group_id, new_levels); let member_table_ids = if group_construct.version - >= CompatibilityVersion::NoMemberTableIds as _ + >= CompatibilityVersion::NoMemberTableIds as i32 { self.state_table_info .compaction_group_member_table_ids(*compaction_group_id) @@ -537,7 +537,8 @@ impl HummockVersion { BTreeSet::from_iter(group_construct.table_ids.clone()) }; - if group_construct.version >= CompatibilityVersion::SplitGroupByTableId as _ + if group_construct.version + >= CompatibilityVersion::SplitGroupByTableId as i32 { let split_key = if group_construct.split_key.is_some() { Some(Bytes::from(group_construct.split_key.clone().unwrap())) From 4aad830f4140aed77c3f6d884c75882996d168d4 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Wed, 18 Dec 2024 17:12:14 +0800 Subject: [PATCH 2/8] fix comm --- src/common/src/types/datetime.rs | 35 ++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs index b412e3c15f637..ccdd6d0cfbaf0 100644 --- a/src/common/src/types/datetime.rs +++ b/src/common/src/types/datetime.rs @@ -494,16 +494,26 @@ impl Timestamp { let secs = cur .read_i64::() .context("failed to read i64 from Time buffer")?; - let nsecs = cur - .read_u32::() - .context("failed to read u32 from Time buffer")?; - - Ok(Timestamp::with_secs_nsecs(secs, nsecs)?) + if Self::has_timestamp_namo_format_state(secs) { + let secs = Self::remove_timestamp_namo_format_state(secs); + let nsecs = cur + .read_u32::() + .context("failed to read u32 from Time buffer")?; + Ok(Timestamp::with_secs_nsecs(secs, nsecs)?) + } else { + Ok(Timestamp::with_micros(secs)?) + } } + // Since timestamp secs is much smaller than i64, we use the highest 2 bit to store the format information, which is compatible with the old format. + // New format: secs(i64) + nsecs(u32) + // Old format: micros(i64) pub fn to_protobuf(self, output: &mut T) -> ArrayResult { let timestamp_size = output - .write(&(self.0.and_utc().timestamp()).to_be_bytes()) + .write( + &(Self::add_timestamp_namo_format_state(self.0.and_utc().timestamp())) + .to_be_bytes(), + ) .map_err(Into::::into)?; let timestamp_subsec_nanos_size = output .write(&(self.0.and_utc().timestamp_subsec_nanos()).to_be_bytes()) @@ -511,6 +521,19 @@ impl Timestamp { Ok(timestamp_subsec_nanos_size + timestamp_size) } + fn add_timestamp_namo_format_state(value: i64) -> i64 { + value ^ (0b01 << 62) + } + + fn has_timestamp_namo_format_state(value: i64) -> bool { + let state = (value >> 62) & 0b11; + state == 0b10 || state == 0b01 + } + + fn remove_timestamp_namo_format_state(value: i64) -> i64 { + value ^ (0b01 << 62) + } + pub fn get_timestamp_nanos(&self) -> i64 { self.0.and_utc().timestamp_nanos_opt().unwrap() } From a6acfc28978ee39ce5deb824215ff545f2513712 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Mon, 23 Dec 2024 14:25:35 +0800 Subject: [PATCH 3/8] add ci --- e2e_test/batch/types/timestamp_ns.slt.part | 112 +++++++++++++++++++++ src/expr/impl/src/scalar/to_char.rs | 2 + 2 files changed, 114 insertions(+) create mode 100644 e2e_test/batch/types/timestamp_ns.slt.part diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part new file mode 100644 index 0000000000000..7b89cc2f69fa1 --- /dev/null +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -0,0 +1,112 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +create table t1(v1 int, v2 timestamp); + +statement ok +insert into t1 values(1,'2013-01-01 01:01:01.123456789'),(2,'2012-01-01 01:01:01.123456'),(3,'0000-01-01 01:01:01.123456789'),(4,'2213-01-01 01:01:01.123456789'),(5,null); + +query T +select * from t1; +---- +3 0001-01-01 01:01:01.123456789 BC +5 null +1 2013-01-01 01:01:01.123456789 +4 2213-01-01 01:01:01.123456789 +2 2012-01-01 01:01:01.123456 + +query T +select * from t1 where v1 is null; +---- +5 null + +query T +select v1, v2, +case + when extract(year from v2) < 2000 then 'Before 2000' + when extract(year from v2) >= 2000 and extract(year from v2) < 2100 then '21st Century' + else 'Future' +end as time_period +from t1; +---- +2 2012-01-01 01:01:01.123456 21st Century +1 2013-01-01 01:01:01.123456789 21st Century +4 2213-01-01 01:01:01.123456789 Future +3 0001-01-01 01:01:01.123456789 BC Before 2000 +5 null Future + +query T +select v1, v2, coalesce(v2, '1900-01-01 00:00:00') as coalesce_v2 from t1; +---- +3 0001-01-01 01:01:01.123456789 BC 0001-01-01 01:01:01.123456789 BC +5 null 1900-01-01 00:00:00 +1 2013-01-01 01:01:01.123456789 2013-01-01 01:01:01.123456789 +4 2213-01-01 01:01:01.123456789 2213-01-01 01:01:01.123456789 +2 2012-01-01 01:01:01.123456 2012-01-01 01:01:01.123456 + +query T +select count(v2) as total_rows from t1; +---- +4 + +query T +select * from t1 order by v2; +---- +3 0001-01-01 01:01:01.123456789 BC +2 2012-01-01 01:01:01.123456 +1 2013-01-01 01:01:01.123456789 +4 2213-01-01 01:01:01.123456789 +5 + +query T +select * from t1 where v2 >= '2012-01-01 01:01:01.123456'; +---- +2 2012-01-01 01:01:01.123456 +1 2013-01-01 01:01:01.123456789 +4 2213-01-01 01:01:01.123456789 + +query T +select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp with time zone) as timestamptz_v2 from t1; +---- +3 0001-01-01 BC 0001-01-01 01:01:01.123456+00:00 BC +5 null null +1 2013-01-01 2013-01-01 01:01:01.123456+00:00 +4 2213-01-01 2213-01-01 01:01:01.123456+00:00 +2 2012-01-01 2012-01-01 01:01:01.123456+00:00 + +query T +select v1, date_trunc('day', v2) AS truncated_v2 from t1; +---- +3 0001-01-01 00:00:00 BC +5 null +2 2012-01-01 00:00:00 +1 2013-01-01 00:00:00 +4 2213-01-01 00:00:00 + +query T +select v1, v2 at time zone 'UTC' as v2_utc from t1; +---- +3 0001-01-01 01:01:01.123456+00:00 BC +5 null +1 2013-01-01 01:01:01.123456+00:00 +4 2213-01-01 01:01:01.123456+00:00 +2 2012-01-01 01:01:01.123456+00:00 + +query T +select v1, to_char(v2, 'YYYY-MM-DD HH24:MI:SS.NS') as formatted_v2 from t1; +---- +3 0001-01-01 01:01:01.123456789 BC +5 null +1 2013-01-01 01:01:01.123456000 +4 2213-01-01 01:01:01.123456789 +2 2012-01-01 01:01:01.123456000 + +query T +select generate_series('2013-01-01 01:01:01.123456789'::timestamp,'2013-01-01 01:01:05.123456790'::timestamp, '1 s'); +---- +2013-01-01 01:01:01.123456789 +2013-01-01 01:01:02.123456789 +2013-01-01 01:01:03.123456789 +2013-01-01 01:01:04.123456789 +2013-01-01 01:01:05.123456789 \ No newline at end of file diff --git a/src/expr/impl/src/scalar/to_char.rs b/src/expr/impl/src/scalar/to_char.rs index 7c9e859675d03..96fad6fc269c5 100644 --- a/src/expr/impl/src/scalar/to_char.rs +++ b/src/expr/impl/src/scalar/to_char.rs @@ -87,6 +87,8 @@ impl ChronoPattern { ("Mon", "%b"), ("DD", "%d"), ("dd", "%d"), + ("NS", "%9f"), + ("ns", "%9f"), ("US", "%6f"), ("us", "%6f"), ("MS", "%3f"), From 026e08a838d35e1c8a6615b3dc1f5d3df1cb8568 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Mon, 23 Dec 2024 15:04:36 +0800 Subject: [PATCH 4/8] fix comm --- e2e_test/batch/types/timestamp_ns.slt.part | 6 +- src/common/src/types/datetime.rs | 71 ++++++++++++------- .../compaction_group/hummock_version_ext.rs | 4 +- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index 7b89cc2f69fa1..18cd67fc0f858 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -57,7 +57,7 @@ select * from t1 order by v2; 2 2012-01-01 01:01:01.123456 1 2013-01-01 01:01:01.123456789 4 2213-01-01 01:01:01.123456789 -5 +5 null query T select * from t1 where v2 >= '2012-01-01 01:01:01.123456'; @@ -70,7 +70,7 @@ query T select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp with time zone) as timestamptz_v2 from t1; ---- 3 0001-01-01 BC 0001-01-01 01:01:01.123456+00:00 BC -5 null null +5 null null 1 2013-01-01 2013-01-01 01:01:01.123456+00:00 4 2213-01-01 2213-01-01 01:01:01.123456+00:00 2 2012-01-01 2012-01-01 01:01:01.123456+00:00 @@ -85,7 +85,7 @@ select v1, date_trunc('day', v2) AS truncated_v2 from t1; 4 2213-01-01 00:00:00 query T -select v1, v2 at time zone 'UTC' as v2_utc from t1; +select v1, v2 at time zone 'UTC' as v2_utc from t1; ---- 3 0001-01-01 01:01:01.123456+00:00 BC 5 null diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs index ccdd6d0cfbaf0..be6cbf9625423 100644 --- a/src/common/src/types/datetime.rs +++ b/src/common/src/types/datetime.rs @@ -481,6 +481,38 @@ impl Time { } } +/// document about old and new format, including the meaning of highest 2 bits, and the corresponding accepted ranges. +/// The enumeration holds the correct value, which will be added(removed) to highest 2 bits when calling `to_protobuf` and `from_protobuf` methods +enum FirstI64 { + V0 { usecs: i64 }, + V1 { secs: i64 }, +} +impl FirstI64 { + pub fn to_protobuf(&self) -> i64 { + match self { + FirstI64::V0 { usecs } => *usecs, + FirstI64::V1 { secs } => secs ^ (0b01 << 62), + } + } + + pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult { + let value = cur + .read_i64::() + .context("failed to read i64 from Time buffer")?; + if Self::is_v1_format_state(value) { + let secs = value ^ (0b01 << 62); + Ok(FirstI64::V1 { secs }) + } else { + Ok(FirstI64::V0 { usecs: value }) + } + } + + fn is_v1_format_state(value: i64) -> bool { + let state = (value >> 62) & 0b11; + state == 0b10 || state == 0b01 + } +} + impl Timestamp { pub fn with_secs_nsecs(secs: i64, nsecs: u32) -> Result { Ok(Timestamp::new({ @@ -491,17 +523,14 @@ impl Timestamp { } pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult { - let secs = cur - .read_i64::() - .context("failed to read i64 from Time buffer")?; - if Self::has_timestamp_namo_format_state(secs) { - let secs = Self::remove_timestamp_namo_format_state(secs); - let nsecs = cur - .read_u32::() - .context("failed to read u32 from Time buffer")?; - Ok(Timestamp::with_secs_nsecs(secs, nsecs)?) - } else { - Ok(Timestamp::with_micros(secs)?) + match FirstI64::from_protobuf(cur)? { + FirstI64::V0 { usecs } => Ok(Timestamp::with_micros(usecs)?), + FirstI64::V1 { secs } => { + let nsecs = cur + .read_u32::() + .context("failed to read u32 from Time buffer")?; + Ok(Timestamp::with_secs_nsecs(secs, nsecs)?) + } } } @@ -511,8 +540,11 @@ impl Timestamp { pub fn to_protobuf(self, output: &mut T) -> ArrayResult { let timestamp_size = output .write( - &(Self::add_timestamp_namo_format_state(self.0.and_utc().timestamp())) - .to_be_bytes(), + &(FirstI64::V1 { + secs: self.0.and_utc().timestamp(), + } + .to_protobuf()) + .to_be_bytes(), ) .map_err(Into::::into)?; let timestamp_subsec_nanos_size = output @@ -521,19 +553,6 @@ impl Timestamp { Ok(timestamp_subsec_nanos_size + timestamp_size) } - fn add_timestamp_namo_format_state(value: i64) -> i64 { - value ^ (0b01 << 62) - } - - fn has_timestamp_namo_format_state(value: i64) -> bool { - let state = (value >> 62) & 0b11; - state == 0b10 || state == 0b01 - } - - fn remove_timestamp_namo_format_state(value: i64) -> i64 { - value ^ (0b01 << 62) - } - pub fn get_timestamp_nanos(&self) -> i64 { self.0.and_utc().timestamp_nanos_opt().unwrap() } diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 591b4f956b008..85d22f4c4a4c1 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -495,7 +495,7 @@ impl HummockVersion { .clone_from(&group_construct.table_ids); self.levels.insert(*compaction_group_id, new_levels); let member_table_ids = if group_construct.version - >= CompatibilityVersion::NoMemberTableIds as i32 + >= CompatibilityVersion::NoMemberTableIds as _ { self.state_table_info .compaction_group_member_table_ids(*compaction_group_id) @@ -509,7 +509,7 @@ impl HummockVersion { }; if group_construct.version - >= CompatibilityVersion::SplitGroupByTableId as i32 + >= CompatibilityVersion::SplitGroupByTableId as _ { let split_key = if group_construct.split_key.is_some() { Some(Bytes::from(group_construct.split_key.clone().unwrap())) From f1cb8ab7e4be06c6af2fcdd04ebca06e5db70b96 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Mon, 23 Dec 2024 15:30:12 +0800 Subject: [PATCH 5/8] fix fmt --- .../hummock_sdk/src/compaction_group/hummock_version_ext.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 85d22f4c4a4c1..6575d69886968 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -508,8 +508,7 @@ impl HummockVersion { BTreeSet::from_iter(group_construct.table_ids.clone()) }; - if group_construct.version - >= CompatibilityVersion::SplitGroupByTableId as _ + if group_construct.version >= CompatibilityVersion::SplitGroupByTableId as _ { let split_key = if group_construct.split_key.is_some() { Some(Bytes::from(group_construct.split_key.clone().unwrap())) From d09d4630c51bceac6caccfd18b591424a4c6089a Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Wed, 25 Dec 2024 13:37:10 +0800 Subject: [PATCH 6/8] add doc + fix ci --- e2e_test/batch/types/timestamp_ns.slt.part | 2 +- src/common/src/types/datetime.rs | 7 +++++-- .../src/compaction_group/hummock_version_ext.rs | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index 18cd67fc0f858..d5fc03385063b 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -98,7 +98,7 @@ select v1, to_char(v2, 'YYYY-MM-DD HH24:MI:SS.NS') as formatted_v2 from t1; ---- 3 0001-01-01 01:01:01.123456789 BC 5 null -1 2013-01-01 01:01:01.123456000 +1 2013-01-01 01:01:01.123456789 4 2213-01-01 01:01:01.123456789 2 2012-01-01 01:01:01.123456000 diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs index be6cbf9625423..99981a83b6dae 100644 --- a/src/common/src/types/datetime.rs +++ b/src/common/src/types/datetime.rs @@ -481,8 +481,11 @@ impl Time { } } -/// document about old and new format, including the meaning of highest 2 bits, and the corresponding accepted ranges. -/// The enumeration holds the correct value, which will be added(removed) to highest 2 bits when calling `to_protobuf` and `from_protobuf` methods +// The first 64 bits of protobuf encoding for `Timestamp` type has 2 possible meanings. +// * When the highest 2 bits are `11` or `00` (i.e. values ranging from `0b1100...00` to `0b0011..11`), +// it is *microseconds* since 1970-01-01 midnight. 2^62 microseconds covers 146235 years. +// * When the highest 2 bits are `10` or `01`, we flip the second bit to get values from `0b1100...00` to `0b0011..11` again. +// It is *seconds* since 1970-01-01 midnight. It is then followed by another 32 bits as nanoseconds within a second. enum FirstI64 { V0 { usecs: i64 }, V1 { secs: i64 }, diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 6575d69886968..591b4f956b008 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -495,7 +495,7 @@ impl HummockVersion { .clone_from(&group_construct.table_ids); self.levels.insert(*compaction_group_id, new_levels); let member_table_ids = if group_construct.version - >= CompatibilityVersion::NoMemberTableIds as _ + >= CompatibilityVersion::NoMemberTableIds as i32 { self.state_table_info .compaction_group_member_table_ids(*compaction_group_id) @@ -508,7 +508,8 @@ impl HummockVersion { BTreeSet::from_iter(group_construct.table_ids.clone()) }; - if group_construct.version >= CompatibilityVersion::SplitGroupByTableId as _ + if group_construct.version + >= CompatibilityVersion::SplitGroupByTableId as i32 { let split_key = if group_construct.split_key.is_some() { Some(Bytes::from(group_construct.split_key.clone().unwrap())) From 2c87c76a9739bcdf584112a59e92bbb25d4b453f Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Thu, 26 Dec 2024 14:56:25 +0800 Subject: [PATCH 7/8] fix ci fxi ci fix ci fix ci --- e2e_test/batch/types/timestamp_ns.slt.part | 74 +++++++++---------- .../compaction_group/hummock_version_ext.rs | 8 +- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index d5fc03385063b..4f42e7a7d3465 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -7,21 +7,21 @@ create table t1(v1 int, v2 timestamp); statement ok insert into t1 values(1,'2013-01-01 01:01:01.123456789'),(2,'2012-01-01 01:01:01.123456'),(3,'0000-01-01 01:01:01.123456789'),(4,'2213-01-01 01:01:01.123456789'),(5,null); -query T +query T rowsort select * from t1; ---- -3 0001-01-01 01:01:01.123456789 BC -5 null 1 2013-01-01 01:01:01.123456789 -4 2213-01-01 01:01:01.123456789 2 2012-01-01 01:01:01.123456 +3 0001-01-01 01:01:01.123456789 BC +4 2213-01-01 01:01:01.123456789 +5 NULL query T -select * from t1 where v1 is null; +select * from t1 where v2 is null; ---- -5 null +5 NULL -query T +query T rowsort select v1, v2, case when extract(year from v2) < 2000 then 'Before 2000' @@ -30,79 +30,79 @@ case end as time_period from t1; ---- -2 2012-01-01 01:01:01.123456 21st Century 1 2013-01-01 01:01:01.123456789 21st Century -4 2213-01-01 01:01:01.123456789 Future +2 2012-01-01 01:01:01.123456 21st Century 3 0001-01-01 01:01:01.123456789 BC Before 2000 -5 null Future +4 2213-01-01 01:01:01.123456789 Future +5 NULL Future -query T +query T rowsort select v1, v2, coalesce(v2, '1900-01-01 00:00:00') as coalesce_v2 from t1; ---- -3 0001-01-01 01:01:01.123456789 BC 0001-01-01 01:01:01.123456789 BC -5 null 1900-01-01 00:00:00 1 2013-01-01 01:01:01.123456789 2013-01-01 01:01:01.123456789 -4 2213-01-01 01:01:01.123456789 2213-01-01 01:01:01.123456789 2 2012-01-01 01:01:01.123456 2012-01-01 01:01:01.123456 +3 0001-01-01 01:01:01.123456789 BC 0001-01-01 01:01:01.123456789 BC +4 2213-01-01 01:01:01.123456789 2213-01-01 01:01:01.123456789 +5 NULL 1900-01-01 00:00:00 query T select count(v2) as total_rows from t1; ---- 4 -query T +query T rowsort select * from t1 order by v2; ---- -3 0001-01-01 01:01:01.123456789 BC -2 2012-01-01 01:01:01.123456 1 2013-01-01 01:01:01.123456789 +2 2012-01-01 01:01:01.123456 +3 0001-01-01 01:01:01.123456789 BC 4 2213-01-01 01:01:01.123456789 -5 null +5 NULL -query T +query T rowsort select * from t1 where v2 >= '2012-01-01 01:01:01.123456'; ---- -2 2012-01-01 01:01:01.123456 1 2013-01-01 01:01:01.123456789 +2 2012-01-01 01:01:01.123456 4 2213-01-01 01:01:01.123456789 -query T +query T rowsort select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp with time zone) as timestamptz_v2 from t1; ---- -3 0001-01-01 BC 0001-01-01 01:01:01.123456+00:00 BC -5 null null 1 2013-01-01 2013-01-01 01:01:01.123456+00:00 -4 2213-01-01 2213-01-01 01:01:01.123456+00:00 2 2012-01-01 2012-01-01 01:01:01.123456+00:00 +3 0001-01-01 BC 0001-01-01 01:01:01.123456+00:00 BC +4 2213-01-01 2213-01-01 01:01:01.123456+00:00 +5 NULL NULL -query T +query T rowsort select v1, date_trunc('day', v2) AS truncated_v2 from t1; ---- -3 0001-01-01 00:00:00 BC -5 null -2 2012-01-01 00:00:00 1 2013-01-01 00:00:00 +2 2012-01-01 00:00:00 +3 0001-01-01 00:00:00 BC 4 2213-01-01 00:00:00 +5 NULL -query T +query T rowsort select v1, v2 at time zone 'UTC' as v2_utc from t1; ---- -3 0001-01-01 01:01:01.123456+00:00 BC -5 null 1 2013-01-01 01:01:01.123456+00:00 -4 2213-01-01 01:01:01.123456+00:00 2 2012-01-01 01:01:01.123456+00:00 +3 0001-01-01 01:01:01.123456+00:00 BC +4 2213-01-01 01:01:01.123456+00:00 +5 NULL -query T +query T rowsort select v1, to_char(v2, 'YYYY-MM-DD HH24:MI:SS.NS') as formatted_v2 from t1; ---- -3 0001-01-01 01:01:01.123456789 BC -5 null 1 2013-01-01 01:01:01.123456789 -4 2213-01-01 01:01:01.123456789 2 2012-01-01 01:01:01.123456000 +3 0000-01-01 01:01:01.123456789 +4 2213-01-01 01:01:01.123456789 +5 NULL -query T +query T rowsort select generate_series('2013-01-01 01:01:01.123456789'::timestamp,'2013-01-01 01:01:05.123456790'::timestamp, '1 s'); ---- 2013-01-01 01:01:01.123456789 diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index 591b4f956b008..a3a79b0d12b3c 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -494,8 +494,8 @@ impl HummockVersion { .member_table_ids .clone_from(&group_construct.table_ids); self.levels.insert(*compaction_group_id, new_levels); - let member_table_ids = if group_construct.version - >= CompatibilityVersion::NoMemberTableIds as i32 + let member_table_ids = if group_construct.version() + >= CompatibilityVersion::NoMemberTableIds { self.state_table_info .compaction_group_member_table_ids(*compaction_group_id) @@ -508,9 +508,7 @@ impl HummockVersion { BTreeSet::from_iter(group_construct.table_ids.clone()) }; - if group_construct.version - >= CompatibilityVersion::SplitGroupByTableId as i32 - { + if group_construct.version() >= CompatibilityVersion::SplitGroupByTableId { let split_key = if group_construct.split_key.is_some() { Some(Bytes::from(group_construct.split_key.clone().unwrap())) } else { From d975bf0fc2635d4cd165452a275a0299ee12cbe3 Mon Sep 17 00:00:00 2001 From: xxhZs <1060434431@qq.com> Date: Fri, 27 Dec 2024 10:37:27 +0800 Subject: [PATCH 8/8] fix ci --- e2e_test/batch/types/timestamp_ns.slt.part | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index 4f42e7a7d3465..080f7d846b4a7 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -109,4 +109,7 @@ select generate_series('2013-01-01 01:01:01.123456789'::timestamp,'2013-01-01 01 2013-01-01 01:01:02.123456789 2013-01-01 01:01:03.123456789 2013-01-01 01:01:04.123456789 -2013-01-01 01:01:05.123456789 \ No newline at end of file +2013-01-01 01:01:05.123456789 + +statement ok +drop table t1; \ No newline at end of file