diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c43a610fa2c24..da99e7a0a0d23 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,6 +22,9 @@ updates: update-types: ["version-update:semver-minor", "version-update:semver-major"] - dependency-name: "parquet" update-types: ["version-update:semver-minor", "version-update:semver-major"] + # bump sqllogictest manually together with sqllogictest-bin in CI docker image + - dependency-name: "sqllogictest" + update-types: ["version-update:semver-minor", "version-update:semver-major"] # Create a group of dependencies to be updated together in one pull request groups: aws: diff --git a/Cargo.lock b/Cargo.lock index b5f04b2a910bb..fee7254124c22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1331,22 +1331,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-http" -version = "0.60.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e4199d5d62ab09be6a64650c06cc5c4aa45806fed4c74bc4a5c8eaf039a6fa" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.9", - "http-body 0.4.5", - "pin-project-lite", - "tracing", -] - [[package]] name = "aws-lc-rs" version = "1.6.2" @@ -1597,7 +1581,7 @@ dependencies = [ "hex", "hmac", "http 0.2.9", - "http 1.1.0", + "http 1.2.0", "once_cell", "p256 0.11.1", "percent-encoding", @@ -1729,7 +1713,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.9", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "tokio", "tracing", @@ -1747,7 +1731,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.9", - "http 1.1.0", + "http 1.2.0", "http-body 0.4.5", "http-body 1.0.0", "http-body-util", @@ -1806,7 +1790,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "hyper 1.4.1", @@ -1839,7 +1823,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "mime", @@ -1859,7 +1843,7 @@ checksum = "077959a7f8cf438676af90b483304528eb7e16eadadb7f44e9ada4f9dceb9e62" dependencies = [ "axum-core", "chrono", - "http 1.1.0", + "http 1.2.0", "mime_guess", "rust-embed", "tower-service", @@ -1875,7 +1859,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "mime", @@ -3506,19 +3490,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core 0.9.8", -] - [[package]] name = "dashmap" version = "6.1.0" @@ -3561,7 +3532,7 @@ dependencies = [ "bytes", "bzip2", "chrono", - "dashmap 6.1.0", + "dashmap", "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", @@ -3654,7 +3625,7 @@ checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" dependencies = [ "arrow 52.2.0", "chrono", - "dashmap 6.1.0", + "dashmap", "datafusion-common", "datafusion-expr", "futures", @@ -4050,7 +4021,7 @@ dependencies = [ "bytes", "cfg-if", "chrono", - "dashmap 6.1.0", + "dashmap", "datafusion", "datafusion-common", "datafusion-expr", @@ -5346,21 +5317,6 @@ dependencies = [ "libc", ] -[[package]] -name = "function_name" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1ab577a896d09940b5fe12ec5ae71f9d8211fff62c919c03a3750a9901e98a7" -dependencies = [ - "function_name-proc-macro", -] - -[[package]] -name = "function_name-proc-macro" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673464e1e314dd67a0fd9544abc99e8eb28d0c7e3b69b033bcff9b2d00b87333" - [[package]] name = "funty" version = "2.0.0" @@ -5646,7 +5602,7 @@ checksum = "cae77099e2399aea466bba05f0d23a150b4f34ed7ce535835e71d91399e65b58" dependencies = [ "anyhow", "async-trait", - "http 1.1.0", + "http 1.2.0", "thiserror 1.0.63", "tokio", "tonic", @@ -5729,7 +5685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3eaaad103912825594d674a4b1e556ccbb05a13a6cac17dcfd871997fb760a" dependencies = [ "google-cloud-token", - "http 1.1.0", + "http 1.2.0", "thiserror 1.0.63", "tokio", "tokio-retry", @@ -5795,14 +5751,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" dependencies = [ "cfg-if", - "dashmap 5.5.3", "futures", "futures-timer", "no-std-compat", "nonzero_ext", "parking_lot 0.12.1", "portable-atomic", - "rand", "smallvec", "spinning_top", ] @@ -5859,7 +5813,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -6058,9 +6012,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -6085,7 +6039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -6096,7 +6050,7 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "pin-project-lite", ] @@ -6159,7 +6113,7 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "httparse", "httpdate", @@ -6194,7 +6148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", - "http 1.1.0", + "http 1.2.0", "hyper 1.4.1", "hyper-util", "rustls 0.22.4", @@ -6255,7 +6209,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "hyper 1.4.1", "pin-project-lite", @@ -6373,7 +6327,7 @@ source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=53f786fb214 dependencies = [ "async-trait", "chrono", - "http 1.1.0", + "http 1.2.0", "iceberg", "itertools 0.13.0", "log", @@ -7365,7 +7319,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "http 1.1.0", + "http 1.2.0", "madsim", "spin 0.9.8", "tracing", @@ -7442,7 +7396,7 @@ checksum = "f271a476bbaa9d2139e1e1a5beb869c6119e805a0b67ad2b2857e4a8785b111a" dependencies = [ "prettyplease 0.2.15", "proc-macro2", - "prost-build 0.13.1", + "prost-build 0.13.4", "quote", "syn 2.0.87", "tonic-build", @@ -8312,7 +8266,7 @@ dependencies = [ "flagset", "futures", "getrandom", - "http 1.1.0", + "http 1.2.0", "log", "md-5", "once_cell", @@ -8344,7 +8298,7 @@ dependencies = [ "flagset", "futures", "getrandom", - "http 1.1.0", + "http 1.2.0", "log", "md-5", "once_cell", @@ -8486,7 +8440,7 @@ checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727" dependencies = [ "async-trait", "futures-core", - "http 1.1.0", + "http 1.2.0", "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", @@ -9594,6 +9548,15 @@ dependencies = [ "prost-derive 0.13.1", ] +[[package]] +name = "prost" +version = "0.13.4" +source = "git+https://github.com/xxchan/prost.git?rev=0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163#0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163" +dependencies = [ + "bytes", + "prost-derive 0.13.4", +] + [[package]] name = "prost-build" version = "0.11.9" @@ -9618,20 +9581,18 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" +version = "0.13.4" +source = "git+https://github.com/xxchan/prost.git?rev=0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163#0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163" dependencies = [ - "bytes", - "heck 0.5.0", - "itertools 0.13.0", + "heck 0.4.1", + "itertools 0.10.5", "log", - "multimap 0.10.0", + "multimap 0.8.3", "once_cell", "petgraph", "prettyplease 0.2.15", - "prost 0.13.1", - "prost-types 0.13.1", + "prost 0.13.4", + "prost-types 0.13.4", "regex", "syn 2.0.87", "tempfile", @@ -9670,7 +9631,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "prost-derive" +version = "0.13.4" +source = "git+https://github.com/xxchan/prost.git?rev=0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163#0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163" +dependencies = [ + "anyhow", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.87", @@ -9719,6 +9692,14 @@ dependencies = [ "prost 0.13.1", ] +[[package]] +name = "prost-types" +version = "0.13.4" +source = "git+https://github.com/xxchan/prost.git?rev=0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163#0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163" +dependencies = [ + "prost 0.13.4", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -10234,7 +10215,7 @@ dependencies = [ "hex", "hmac", "home", - "http 1.1.0", + "http 1.2.0", "jsonwebtoken", "log", "once_cell", @@ -10306,7 +10287,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "hyper 1.4.1", @@ -10353,7 +10334,7 @@ checksum = "a45d100244a467870f6cb763c4484d010a6bed6bd610b3676e3825d93fb4cfbd" dependencies = [ "anyhow", "async-trait", - "http 1.1.0", + "http 1.2.0", "reqwest 0.12.4", "serde", "thiserror 1.0.63", @@ -10455,7 +10436,6 @@ dependencies = [ "regex", "reqwest 0.12.4", "serde", - "serde_json", "serde_with 3.8.1", "serde_yaml", "sqlx", @@ -10499,7 +10479,6 @@ version = "2.3.0-alpha" dependencies = [ "anyhow", "async-trait", - "bincode 1.3.3", "bytes", "itertools 0.13.0", "parking_lot 0.12.1", @@ -10522,7 +10501,6 @@ dependencies = [ "anyhow", "async-recursion", "async-trait", - "criterion", "either", "futures", "futures-async-stream", @@ -10552,7 +10530,6 @@ dependencies = [ "tempfile", "thiserror 1.0.63", "thiserror-ext", - "tikv-jemallocator", "tokio-postgres", "tokio-stream 0.1.15", "tracing", @@ -10654,7 +10631,6 @@ version = "2.3.0-alpha" dependencies = [ "clap", "madsim-tokio", - "prometheus", "risingwave_batch_executors", "risingwave_common", "risingwave_compactor", @@ -10673,14 +10649,12 @@ dependencies = [ name = "risingwave_cmd_all" version = "2.3.0-alpha" dependencies = [ - "anyhow", "clap", "console", "const-str", "expect-test", "home", "madsim-tokio", - "prometheus", "risingwave_batch_executors", "risingwave_cmd", "risingwave_common", @@ -10745,10 +10719,9 @@ dependencies = [ "governor", "hashbrown 0.14.5", "hex", - "http 1.1.0", + "http 1.2.0", "http-body 0.4.5", "humantime", - "hytra", "itertools 0.13.0", "itoa", "jsonbb", @@ -10829,7 +10802,6 @@ dependencies = [ "ethnum", "fixedbitset 0.5.0", "jsonbb", - "lru 0.7.6", "risingwave_common_proc_macro", "rust_decimal", "serde_json", @@ -10839,7 +10811,6 @@ dependencies = [ name = "risingwave_common_heap_profiling" version = "2.3.0-alpha" dependencies = [ - "anyhow", "chrono", "madsim-tokio", "parking_lot 0.12.1", @@ -10862,7 +10833,7 @@ dependencies = [ "easy-ext", "futures", "http 0.2.9", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "hyper 0.14.27", "hyper 1.4.1", @@ -10924,7 +10895,7 @@ dependencies = [ "axum", "axum-extra", "futures", - "http 1.1.0", + "http 1.2.0", "madsim-tokio", "madsim-tonic", "prometheus", @@ -10945,21 +10916,14 @@ name = "risingwave_compaction_test" version = "2.3.0-alpha" dependencies = [ "anyhow", - "async-trait", "bytes", "clap", "foyer", - "futures", "madsim-tokio", - "prometheus", - "rand", "risingwave_common", "risingwave_compactor", "risingwave_hummock_sdk", - "risingwave_hummock_test", - "risingwave_meta", "risingwave_meta_node", - "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "risingwave_rt", @@ -10978,7 +10942,6 @@ dependencies = [ "jsonbb", "madsim-tokio", "madsim-tonic", - "parking_lot 0.12.1", "prost 0.13.1", "risingwave_common", "risingwave_common_heap_profiling", @@ -11005,8 +10968,7 @@ dependencies = [ "foyer", "futures", "futures-async-stream", - "http 1.1.0", - "hyper 1.4.1", + "http 1.2.0", "itertools 0.13.0", "madsim-tokio", "madsim-tonic", @@ -11036,7 +10998,6 @@ dependencies = [ "tokio-stream 0.1.15", "tower 0.5.0", "tracing", - "uuid", "workspace-hack", ] @@ -11090,7 +11051,6 @@ dependencies = [ "google-cloud-googleapis", "google-cloud-pubsub", "governor", - "http 0.2.9", "iceberg", "iceberg-catalog-glue", "iceberg-catalog-rest", @@ -11098,8 +11058,6 @@ dependencies = [ "indexmap 2.7.0", "itertools 0.13.0", "jni", - "jsonbb", - "jsonwebtoken", "madsim-rdkafka", "madsim-tokio", "madsim-tonic", @@ -11134,7 +11092,6 @@ dependencies = [ "risingwave_common_estimate_size", "risingwave_connector_codec", "risingwave_jni_core", - "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "rumqttc", @@ -11194,7 +11151,7 @@ dependencies = [ "madsim-tokio", "num-bigint", "prost 0.13.1", - "prost-build 0.13.1", + "prost-build 0.13.4", "prost-reflect", "prost-types 0.13.1", "protox", @@ -11225,7 +11182,6 @@ dependencies = [ "itertools 0.13.0", "madsim-tokio", "madsim-tonic", - "memcomparable", "prost 0.13.1", "regex", "risingwave_common", @@ -11234,7 +11190,6 @@ dependencies = [ "risingwave_hummock_sdk", "risingwave_meta", "risingwave_meta_model", - "risingwave_meta_model_migration", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", @@ -11256,21 +11211,15 @@ name = "risingwave_dml" version = "2.3.0-alpha" dependencies = [ "assert_matches", - "criterion", "futures", "futures-async-stream", "itertools 0.13.0", "madsim-tokio", "parking_lot 0.12.1", "paste", - "rand", "risingwave_common", - "risingwave_connector", - "risingwave_pb", - "rw_futures_util", "tempfile", "thiserror 1.0.63", - "thiserror-ext", "tracing", "workspace-hack", ] @@ -11296,7 +11245,6 @@ version = "2.3.0-alpha" dependencies = [ "anyhow", "bincode 1.3.3", - "bytes", "easy-ext", "madsim-tonic", "serde", @@ -11314,7 +11262,6 @@ dependencies = [ "async-trait", "auto_impl", "await-tree", - "cfg-or-panic", "chrono", "const-currying", "downcast-rs", @@ -11476,7 +11423,6 @@ dependencies = [ "risingwave_expr_impl", "risingwave_frontend_macro", "risingwave_hummock_sdk", - "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "risingwave_sqlparser", @@ -11500,7 +11446,6 @@ dependencies = [ "url", "uuid", "workspace-hack", - "zstd 0.13.0", ] [[package]] @@ -11518,16 +11463,13 @@ version = "2.3.0-alpha" dependencies = [ "bincode 2.0.0-rc.3", "bytes", - "easy-ext", "hex", "itertools 0.13.0", "parse-display", - "prost 0.13.1", "risingwave_common", "risingwave_common_estimate_size", "risingwave_pb", "serde", - "serde_bytes", "tracing", "workspace-hack", ] @@ -11559,7 +11501,6 @@ dependencies = [ "risingwave_rpc_client", "risingwave_storage", "risingwave_test_runner", - "serde", "serial_test", "sync-point", "workspace-hack", @@ -11622,21 +11563,14 @@ dependencies = [ "cfg-or-panic", "chrono", "expect-test", - "foyer", "fs-err", "futures", - "itertools 0.13.0", "jni", "madsim-tokio", "paste", "prost 0.13.1", "risingwave_common", - "risingwave_expr", - "risingwave_hummock_sdk", "risingwave_pb", - "rw_futures_util", - "serde", - "serde_json", "thiserror 1.0.63", "thiserror-ext", "tracing", @@ -11661,15 +11595,10 @@ dependencies = [ name = "risingwave_mem_table_spill_test" version = "2.3.0-alpha" dependencies = [ - "async-trait", - "bytes", - "futures", - "futures-async-stream", "madsim-tokio", "risingwave_common", "risingwave_hummock_sdk", "risingwave_hummock_test", - "risingwave_storage", "risingwave_stream", ] @@ -11681,10 +11610,8 @@ dependencies = [ "arc-swap", "assert_matches", "async-trait", - "aws-config", "axum", "base64-url", - "bincode 1.3.3", "bytes", "chrono", "clap", @@ -11696,19 +11623,14 @@ dependencies = [ "enum-as-inner 0.6.0", "expect-test", "fail", - "flate2", - "function_name", "futures", "hex", - "http 1.1.0", + "http 1.2.0", "itertools 0.13.0", "jsonbb", "madsim-tokio", "madsim-tonic", "maplit", - "memcomparable", - "mime_guess", - "moka", "notify", "num-integer", "num-traits", @@ -11732,7 +11654,6 @@ dependencies = [ "risingwave_rpc_client", "risingwave_sqlparser", "risingwave_test_runner", - "rw-aws-sdk-ec2", "rw_futures_util", "scopeguard", "sea-orm", @@ -11749,7 +11670,6 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", - "url", "uuid", "workspace-hack", ] @@ -11781,7 +11701,6 @@ version = "2.3.0-alpha" dependencies = [ "prost 0.13.1", "risingwave_common", - "risingwave_hummock_sdk", "risingwave_pb", "sea-orm", "serde", @@ -11805,13 +11724,9 @@ dependencies = [ name = "risingwave_meta_node" version = "2.3.0-alpha" dependencies = [ - "anyhow", "clap", "educe", - "either", - "futures", "hex", - "itertools 0.13.0", "madsim-tokio", "madsim-tonic", "otlp-embedded", @@ -11822,12 +11737,10 @@ dependencies = [ "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_meta", - "risingwave_meta_model_migration", "risingwave_meta_service", "risingwave_pb", "risingwave_rpc_client", "sea-orm", - "serde", "serde_json", "thiserror-ext", "tracing", @@ -11840,12 +11753,10 @@ version = "2.3.0-alpha" dependencies = [ "anyhow", "async-trait", - "either", "futures", "itertools 0.13.0", "madsim-tokio", "madsim-tonic", - "prost 0.13.1", "rand", "regex", "risingwave_common", @@ -11856,7 +11767,6 @@ dependencies = [ "risingwave_pb", "sea-orm", "serde_json", - "sync-point", "thiserror-ext", "tokio-stream 0.1.15", "tracing", @@ -11877,12 +11787,10 @@ dependencies = [ "aws-smithy-types", "bytes", "crc32fast", - "either", "fail", "futures", "hyper 0.14.27", "hyper-rustls 0.24.2", - "hyper-tls 0.5.0", "itertools 0.13.0", "madsim", "madsim-aws-sdk-s3", @@ -11910,10 +11818,11 @@ dependencies = [ "pbjson", "pbjson-build", "prost 0.13.1", - "prost-build 0.13.1", + "prost-build 0.13.4", "prost-helpers", "risingwave_error", "serde", + "static_assertions", "strum 0.26.3", "thiserror 1.0.63", "walkdir", @@ -11965,10 +11874,7 @@ dependencies = [ "easy-ext", "either", "futures", - "h2 0.4.7", - "http 1.1.0", - "hyper 1.4.1", - "itertools 0.13.0", + "http 1.2.0", "lru 0.7.6", "madsim-tokio", "madsim-tonic", @@ -11987,7 +11893,6 @@ dependencies = [ "tokio-stream 0.1.15", "tower 0.5.0", "tracing", - "url", "workspace-hack", ] @@ -12039,14 +11944,12 @@ dependencies = [ "itertools 0.13.0", "lru 0.7.6", "madsim", - "madsim-aws-sdk-s3", "madsim-rdkafka", "madsim-tokio", "maplit", "paste", "pin-project", "pretty_assertions", - "prometheus", "rand", "rand_chacha", "risingwave_batch_executors", @@ -12062,7 +11965,6 @@ dependencies = [ "risingwave_meta_node", "risingwave_object_store", "risingwave_pb", - "risingwave_rpc_client", "risingwave_sqlparser", "risingwave_sqlsmith", "serde", @@ -12072,7 +11974,6 @@ dependencies = [ "tempfile", "tikv-jemallocator", "tokio-postgres", - "tokio-stream 0.1.15", "tracing", "tracing-subscriber", "uuid", @@ -12117,7 +12018,6 @@ dependencies = [ "risingwave_expr", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_pb", "risingwave_sqlparser", "similar", "thiserror-ext", @@ -12135,7 +12035,6 @@ dependencies = [ "clap", "futures", "madsim-tokio", - "prometheus", "regex", "risingwave_rt", "serde", @@ -12151,7 +12050,6 @@ dependencies = [ name = "risingwave_storage" version = "2.3.0-alpha" dependencies = [ - "ahash 0.8.11", "anyhow", "arc-swap", "async-trait", @@ -12160,9 +12058,7 @@ dependencies = [ "bincode 1.3.3", "bytes", "criterion", - "crossbeam", "darwin-libproc", - "dashmap 6.1.0", "dyn-clone", "either", "enum-as-inner 0.6.0", @@ -12277,7 +12173,6 @@ dependencies = [ "serde_yaml", "smallvec", "static_assertions", - "strum 0.26.3", "strum_macros 0.26.4", "thiserror 1.0.63", "thiserror-ext", @@ -12299,10 +12194,8 @@ dependencies = [ "prost 0.13.1", "reqwest 0.12.4", "risingwave_pb", - "thiserror 1.0.63", "thiserror-ext", "tracing", - "uuid", ] [[package]] @@ -12726,30 +12619,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "rw-aws-sdk-ec2" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80dba3602b267a7f9dcc546ccbf1d05752447773146253c7e344e2a320630b7f" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.9", - "regex", - "tracing", -] - [[package]] name = "rw_futures_util" version = "0.0.0" @@ -13775,9 +13644,9 @@ dependencies = [ [[package]] name = "sqllogictest" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec31dce96f489e2247a165837f49bbce4912b0cbcf127b79b4fdd87503022ae9" +checksum = "48c03edcabfda1ab894cc63a115b9f014bfc6875916b850ab7498d3cb92daed9" dependencies = [ "async-trait", "educe", @@ -14768,7 +14637,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "httparse", "rand", "ring 0.17.5", @@ -14862,7 +14731,7 @@ dependencies = [ "bytes", "flate2", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "hyper 1.4.1", @@ -14891,7 +14760,7 @@ checksum = "fe4ee8877250136bd7e3d2331632810a4df4ea5e004656990d8d66d2f5ee8a67" dependencies = [ "prettyplease 0.2.15", "proc-macro2", - "prost-build 0.13.1", + "prost-build 0.13.4", "quote", "syn 2.0.87", ] @@ -14941,7 +14810,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.0", "http-body-util", "http-range-header", @@ -16564,7 +16433,6 @@ dependencies = [ name = "with_options" version = "2.3.0-alpha" dependencies = [ - "proc-macro2", "quote", "syn 2.0.87", ] diff --git a/Cargo.toml b/Cargo.toml index 4a86290bbc128..c7177b037f736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,19 @@ keywords = ["sql", "database", "streaming"] license = "Apache-2.0" repository = "https://github.com/risingwavelabs/risingwave" +# some dependencies are not worth deleting. +[workspace.metadata.cargo-machete] +ignored = [ + "workspace-hack", + "expect-test", + "pretty_assertions", + "serde", + "serde_json", +] +[workspace.metadata.cargo-udeps.ignore] +normal = ["workspace-hack"] +development = ["expect-test", "pretty_assertions"] + [workspace.dependencies] foyer = { version = "0.13.1", features = ["tracing", "nightly", "prometheus"] } apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "25113ba88234a9ae23296e981d8302c290fdaa4b", features = [ @@ -107,11 +120,6 @@ aws-sdk-s3 = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -# To bump the version of aws-sdk-ec2, check the README of https://github.com/risingwavelabs/rw-aws-sdk-ec2 -aws-sdk-ec2 = { package = "rw-aws-sdk-ec2", version = "1", default-features = false, features = [ - "rt-tokio", - "rustls", -] } aws-sdk-sqs = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", @@ -204,6 +212,7 @@ tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055" tokio-util = "0.7" tracing-opentelemetry = "0.25" rand = { version = "0.8", features = ["small_rng"] } +governor = { version = "0.6", default-features = false, features = ["std"] } risingwave_backup = { path = "./src/storage/backup" } risingwave_batch = { path = "./src/batch" } risingwave_batch_executors = { path = "./src/batch/executors" } @@ -366,6 +375,8 @@ sqlx = { git = "https://github.com/madsim-rs/sqlx.git", rev = "3efe6d0065963db2a futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" } # patch to remove preserve_order from serde_json bson = { git = "https://github.com/risingwavelabs/bson-rust", rev = "e5175ec" } +# TODO: unpatch after PR merged https://github.com/tokio-rs/prost/pull/1210 +prost-build = { git = "https://github.com/xxchan/prost.git", rev = "0eb1c7b09976cf6b5231e4b8d87bb5908ae6a163" } [workspace.metadata.dylint] libraries = [{ path = "./lints" }] diff --git a/Makefile.toml b/Makefile.toml index 89814dbe36ae5..6079a69e6ab4d 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -382,11 +382,7 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$ [tasks.post-build-risingwave] category = "RiseDev - Build" description = "Copy RisingWave binaries to bin" -condition = { env_set = [ - "ENABLE_BUILD_RUST", -], env_not_set = [ - "USE_SYSTEM_RISINGWAVE", -] } +condition = { env_set = ["ENABLE_BUILD_RUST"], env_not_set = ["USE_SYSTEM_RISINGWAVE"] } dependencies = [ "link-all-in-one-binaries", "link-user-bin", @@ -649,13 +645,23 @@ echo " $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)" [tasks.psql] category = "RiseDev - Start/Stop" -description = "Run local psql client with default connection parameters. You can pass extra arguments to psql." +description = "Run local psql client for RisingWave with default connection parameters. You can pass extra arguments to psql." dependencies = ["check-and-load-risedev-env-file"] script = ''' #!/usr/bin/env bash psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev "$@" ''' +[tasks.pgpsql] +category = "RiseDev - Start/Stop" +description = "Run local psql client for postgres with default connection parameters. You can pass extra arguments to psql." +dependencies = ["check-and-load-risedev-env-file"] +script = ''' +#!/usr/bin/env bash +source "${PREFIX_CONFIG}/risedev-env" +PGHOST=$PGHOST PGPORT=$PGPORT PGUSER=$PGUSER PGDATABASE=$PGDATABASE psql "$@" +''' + [tasks.ctl] category = "RiseDev - Start/Stop" description = "Start RiseCtl" @@ -863,11 +869,6 @@ script = """ #!/usr/bin/env bash set -e -echo "Running Planner Test requires larger stack size, setting RUST_MIN_STACK to 8388608 (8MB) as default." -if [[ -z "${RUST_MIN_STACK}" ]]; then - export RUST_MIN_STACK=8388608 -fi - cargo nextest run --workspace --exclude risingwave_simulation "$@" """ description = "🌟 Run unit tests" @@ -1208,7 +1209,7 @@ script = """ echo "Running $(tput setaf 4)cargo udeps$(tput sgr0) checks" -cargo udeps --workspace --all-targets ${RISINGWAVE_FEATURE_FLAGS} --exclude workspace-hack --exclude risingwave_bench --exclude risingwave_udf --exclude risingwave_simulation +cargo udeps --workspace --all-targets ${RISINGWAVE_FEATURE_FLAGS} --exclude workspace-hack --exclude risingwave_bench --exclude risingwave_simulation """ [tasks.check-trailing-spaces] @@ -1303,7 +1304,7 @@ echo If you still feel this is not enough, you may copy $(tput setaf 4)risedev$( [tasks.ci-start] category = "RiseDev - CI" dependencies = ["clean-data", "pre-start-dev"] -command = "target/debug/risedev-dev" # `risedev-dev` is always built in dev profile +command = "target/debug/risedev-dev" # `risedev-dev` is always built in dev profile env = { RISEDEV_CLEAN_START = true } args = ["${@}"] description = "Clean data and start a full RisingWave dev cluster using risedev-dev" @@ -1340,7 +1341,7 @@ echo "All processes has exited." [tasks.slt] category = "RiseDev - Test - SQLLogicTest" -install_crate = { min_version = "0.23.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [ +install_crate = { min_version = "0.24.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [ "--help", ], install_command = "binstall" } dependencies = ["check-and-load-risedev-env-file"] diff --git a/ci/Dockerfile b/ci/Dockerfile index 21b9c30c678b0..7478d24b66edc 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -70,7 +70,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash RUN cargo binstall -y --locked --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \ cargo-make@0.37.9 \ - sqllogictest-bin@0.23.1 \ + sqllogictest-bin@0.24.0 \ sccache@0.7.4 \ && cargo cache -a \ && rm -rf "/root/.cargo/registry/index" \ diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index 86392d2ecd7d8..969239dea82c2 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -10,7 +10,7 @@ cat ../rust-toolchain # shellcheck disable=SC2155 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml -export BUILD_ENV_VERSION=v20241213 +export BUILD_ENV_VERSION=v20241224 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 7ef17777b390c..4b8d795c142f5 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -90,7 +90,7 @@ services: retries: 5 source-test-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 depends_on: - mysql - mysql-meta @@ -106,7 +106,7 @@ services: - ..:/risingwave sink-test-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 depends_on: - mysql - mysql-meta @@ -129,13 +129,13 @@ services: - ..:/risingwave rw-build-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 volumes: - ..:/risingwave # Standard environment for CI, including MySQL and Postgres for metadata. ci-standard-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 depends_on: - mysql-meta - db @@ -143,14 +143,14 @@ services: - ..:/risingwave iceberg-engine-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 depends_on: - db volumes: - ..:/risingwave ci-flamegraph-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 # NOTE(kwannoel): This is used in order to permit # syscalls for `nperf` (perf_event_open), # so it can do CPU profiling. @@ -161,7 +161,7 @@ services: - ..:/risingwave regress-test-env: - image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241213 + image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241224 depends_on: db: condition: service_healthy diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh index b3e09bd607b28..9d807070ca57a 100755 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -16,12 +16,13 @@ export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud export NEXTEST_HIDE_PROGRESS_BAR=true export RW_TELEMETRY_TYPE=test export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef0123456789abcdef" -export RUST_MIN_STACK=4194304 unset LANG function dump_diagnose_info() { - ./risedev diagnose || true + if [ -f .risingwave/config/risedev-env ]; then + ./risedev diagnose || true + fi } trap dump_diagnose_info EXIT diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 94ceaaaa853c1..20a06a18de7a4 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -144,7 +144,6 @@ risedev ci-kill export RISINGWAVE_CI=true echo "--- e2e, ci-kafka-plus-pubsub, legacy kafka tests" -export RUST_MIN_STACK=4194304 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ risedev ci-start ci-kafka ./e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh diff --git a/ci/scripts/pr-unit-test.sh b/ci/scripts/pr-unit-test.sh index 73b50909802bf..25f11f446206e 100755 --- a/ci/scripts/pr-unit-test.sh +++ b/ci/scripts/pr-unit-test.sh @@ -5,8 +5,4 @@ set -euo pipefail source ci/scripts/common.sh source ci/scripts/pr.env.sh - -# Set RUST_MIN_STACK to 8MB to avoid stack overflow in planner test. -# This is a Unit Test specific setting. -export RUST_MIN_STACK=8388608 ./ci/scripts/run-unit-test.sh diff --git a/dashboard/components/FragmentGraph.tsx b/dashboard/components/FragmentGraph.tsx index 1e7d80c8641c3..2001cbf79c6fd 100644 --- a/dashboard/components/FragmentGraph.tsx +++ b/dashboard/components/FragmentGraph.tsx @@ -24,8 +24,14 @@ import { layoutItem, } from "../lib/layout" import { PlanNodeDatum } from "../pages/fragment_graph" +import { FragmentStats } from "../proto/gen/monitor_service" import { StreamNode } from "../proto/gen/stream_plan" -import { backPressureColor, backPressureWidth } from "./utils/backPressure" +import { + backPressureColor, + backPressureWidth, + epochToUnixMillis, + latencyToColor, +} from "./utils/backPressure" const ReactJson = loadable(() => import("react-json-view")) @@ -95,11 +101,13 @@ export default function FragmentGraph({ fragmentDependency, selectedFragmentId, backPressures, + fragmentStats, }: { planNodeDependencies: Map> fragmentDependency: FragmentBox[] selectedFragmentId?: string backPressures?: Map + fragmentStats?: { [fragmentId: number]: FragmentStats } }) { const svgRef = useRef(null) @@ -188,6 +196,7 @@ export default function FragmentGraph({ useEffect(() => { if (fragmentLayout) { + const now_ms = Date.now() const svgNode = svgRef.current const svgSelection = d3.select(svgNode) @@ -246,13 +255,69 @@ export default function FragmentGraph({ .attr("height", ({ height }) => height - fragmentMarginY * 2) .attr("x", fragmentMarginX) .attr("y", fragmentMarginY) - .attr("fill", "white") + .attr( + "fill", + fragmentStats + ? ({ id }) => { + const fragmentId = parseInt(id) + if (isNaN(fragmentId) || !fragmentStats[fragmentId]) { + return "white" + } + let currentMs = epochToUnixMillis( + fragmentStats[fragmentId].currentEpoch + ) + return latencyToColor(now_ms - currentMs, "white") + } + : "white" + ) .attr("stroke-width", ({ id }) => (isSelected(id) ? 3 : 1)) .attr("rx", 5) .attr("stroke", ({ id }) => isSelected(id) ? theme.colors.blue[500] : theme.colors.gray[500] ) + const getTooltipContent = (id: string) => { + const fragmentId = parseInt(id) + const stats = fragmentStats?.[fragmentId] + const latencySeconds = stats + ? ((now_ms - epochToUnixMillis(stats.currentEpoch)) / 1000).toFixed( + 2 + ) + : "N/A" + const epoch = stats?.currentEpoch ?? "N/A" + + return `Fragment ${fragmentId}
Epoch: ${epoch}
Latency: ${latencySeconds} seconds` + } + + boundingBox + .on("mouseover", (event, { id }) => { + // Remove existing tooltip if any + d3.selectAll(".tooltip").remove() + + // Create new tooltip + d3.select("body") + .append("div") + .attr("class", "tooltip") + .style("position", "absolute") + .style("background", "white") + .style("padding", "10px") + .style("border", "1px solid #ddd") + .style("border-radius", "4px") + .style("pointer-events", "none") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + .style("font-size", "12px") + .html(getTooltipContent(id)) + }) + .on("mousemove", (event) => { + d3.select(".tooltip") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + }) + .on("mouseout", () => { + d3.selectAll(".tooltip").remove() + }) + // Stream node edges let edgeSelection = gSel.select(".edges") if (edgeSelection.empty()) { @@ -409,24 +474,39 @@ export default function FragmentGraph({ .attr("stroke-width", width) .attr("stroke", color) - // Tooltip for back pressure rate - let title = gSel.select("title") - if (title.empty()) { - title = gSel.append("title") - } - - const text = (d: Edge) => { - if (backPressures) { - let value = backPressures.get(`${d.target}_${d.source}`) - if (value) { - return `${value.toFixed(2)}%` + path + .on("mouseover", (event, d) => { + // Remove existing tooltip if any + d3.selectAll(".tooltip").remove() + + if (backPressures) { + const value = backPressures.get(`${d.target}_${d.source}`) + if (value) { + // Create new tooltip + d3.select("body") + .append("div") + .attr("class", "tooltip") + .style("position", "absolute") + .style("background", "white") + .style("padding", "10px") + .style("border", "1px solid #ddd") + .style("border-radius", "4px") + .style("pointer-events", "none") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + .style("font-size", "12px") + .html(`BP: ${value.toFixed(2)}%`) + } } - } - - return "" - } - - title.text(text) + }) + .on("mousemove", (event) => { + d3.select(".tooltip") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + }) + .on("mouseout", () => { + d3.selectAll(".tooltip").remove() + }) return gSel } diff --git a/dashboard/components/RelationGraph.tsx b/dashboard/components/RelationGraph.tsx index a61a6924a370e..59c87275b43ed 100644 --- a/dashboard/components/RelationGraph.tsx +++ b/dashboard/components/RelationGraph.tsx @@ -33,8 +33,14 @@ import { flipLayoutRelation, generateRelationEdges, } from "../lib/layout" +import { RelationStats } from "../proto/gen/monitor_service" import { CatalogModal, useCatalogModal } from "./CatalogModal" -import { backPressureColor, backPressureWidth } from "./utils/backPressure" +import { + backPressureColor, + backPressureWidth, + epochToUnixMillis, + latencyToColor, +} from "./utils/backPressure" function boundBox( relationPosition: RelationPointPosition[], @@ -62,11 +68,13 @@ export default function RelationGraph({ selectedId, setSelectedId, backPressures, + relationStats, }: { nodes: RelationPoint[] selectedId: string | undefined setSelectedId: (id: string) => void backPressures?: Map // relationId-relationId->back_pressure_rate}) + relationStats: { [relationId: number]: RelationStats } | undefined }) { const [modalData, setModalId] = useCatalogModal(nodes.map((n) => n.relation)) @@ -99,6 +107,7 @@ export default function RelationGraph({ const { layoutMap, width, height, links } = layoutMapCallback() useEffect(() => { + const now_ms = Date.now() const svgNode = svgRef.current const svgSelection = d3.select(svgNode) @@ -150,24 +159,39 @@ export default function RelationGraph({ isSelected(d.source) || isSelected(d.target) ? 1 : 0.5 ) - // Tooltip for back pressure rate - let title = sel.select("title") - if (title.empty()) { - title = sel.append("title") - } - - const text = (d: Edge) => { - if (backPressures) { - let value = backPressures.get(`${d.target}_${d.source}`) - if (value) { - return `${value.toFixed(2)}%` + sel + .on("mouseover", (event, d) => { + // Remove existing tooltip if any + d3.selectAll(".tooltip").remove() + + if (backPressures) { + const value = backPressures.get(`${d.target}_${d.source}`) + if (value) { + // Create new tooltip + d3.select("body") + .append("div") + .attr("class", "tooltip") + .style("position", "absolute") + .style("background", "white") + .style("padding", "10px") + .style("border", "1px solid #ddd") + .style("border-radius", "4px") + .style("pointer-events", "none") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + .style("font-size", "12px") + .html(`BP: ${value.toFixed(2)}%`) + } } - } - - return "" - } - - title.text(text) + }) + .on("mousemove", (event) => { + d3.select(".tooltip") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + }) + .on("mouseout", () => { + d3.selectAll(".tooltip").remove() + }) return sel } @@ -189,9 +213,19 @@ export default function RelationGraph({ circle.attr("r", nodeRadius).attr("fill", ({ id, relation }) => { const weight = relationIsStreamingJob(relation) ? "500" : "400" - return isSelected(id) + const baseColor = isSelected(id) ? theme.colors.blue[weight] : theme.colors.gray[weight] + if (relationStats) { + const relationId = parseInt(id) + if (!isNaN(relationId) && relationStats[relationId]) { + const currentMs = epochToUnixMillis( + relationStats[relationId].currentEpoch + ) + return latencyToColor(now_ms - currentMs, baseColor) + } + } + return baseColor }) // Relation name @@ -233,16 +267,50 @@ export default function RelationGraph({ .attr("font-size", 16) .attr("font-weight", "bold") - // Relation type tooltip - let typeTooltip = g.select("title") - if (typeTooltip.empty()) { - typeTooltip = g.append("title") + // Tooltip + const getTooltipContent = (relation: Relation, id: string) => { + const relationId = parseInt(id) + const stats = relationStats?.[relationId] + const latencySeconds = stats + ? ( + (Date.now() - epochToUnixMillis(stats.currentEpoch)) / + 1000 + ).toFixed(2) + : "N/A" + const epoch = stats?.currentEpoch ?? "N/A" + + return `${relation.name} (${relationTypeTitleCase( + relation + )})
Epoch: ${epoch}
Latency: ${latencySeconds} seconds` } - typeTooltip.text( - ({ relation }) => - `${relation.name} (${relationTypeTitleCase(relation)})` - ) + g.on("mouseover", (event, { relation, id }) => { + // Remove existing tooltip if any + d3.selectAll(".tooltip").remove() + + // Create new tooltip + d3.select("body") + .append("div") + .attr("class", "tooltip") + .style("position", "absolute") + .style("background", "white") + .style("padding", "10px") + .style("border", "1px solid #ddd") + .style("border-radius", "4px") + .style("pointer-events", "none") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + .style("font-size", "12px") + .html(getTooltipContent(relation, id)) + }) + .on("mousemove", (event) => { + d3.select(".tooltip") + .style("left", event.pageX + 10 + "px") + .style("top", event.pageY + 10 + "px") + }) + .on("mouseout", () => { + d3.selectAll(".tooltip").remove() + }) // Relation modal g.style("cursor", "pointer").on("click", (_, { relation, id }) => { @@ -265,7 +333,15 @@ export default function RelationGraph({ nodeSelection.enter().call(createNode) nodeSelection.call(applyNode) nodeSelection.exit().remove() - }, [layoutMap, links, selectedId, setModalId, setSelectedId, backPressures]) + }, [ + layoutMap, + links, + selectedId, + setModalId, + setSelectedId, + backPressures, + relationStats, + ]) return ( <> diff --git a/dashboard/components/utils/backPressure.tsx b/dashboard/components/utils/backPressure.tsx index afb26e0746da4..841e64baafa82 100644 --- a/dashboard/components/utils/backPressure.tsx +++ b/dashboard/components/utils/backPressure.tsx @@ -41,3 +41,47 @@ export function backPressureWidth(value: number, scale: number) { return scale * (value / 100) + 2 } + +export function epochToUnixMillis(epoch: number) { + // UNIX_RISINGWAVE_DATE_SEC + return 1617235200000 + epoch / 65536 +} + +export function latencyToColor(latency_ms: number, baseColor: string) { + const LOWER = 10000 // 10s + const UPPER = 300000 // 5min + + const colorRange = [ + baseColor, + theme.colors.yellow["200"], + theme.colors.orange["300"], + theme.colors.red["400"], + ].map((c) => tinycolor(c)) + + if (latency_ms <= LOWER) { + return baseColor + } + + if (latency_ms >= UPPER) { + return theme.colors.red["400"] + } + + // Map log(latency) to [0,1] range between 10s and 5min + const minLog = Math.log(LOWER) + const maxLog = Math.log(UPPER) + const latencyLog = Math.log(latency_ms) + const normalizedPos = (latencyLog - minLog) / (maxLog - minLog) + + // Map to color range index + const step = colorRange.length - 1 + const pos = normalizedPos * step + const floor = Math.floor(pos) + const ceil = Math.ceil(pos) + + // Interpolate between colors + const color = tinycolor(colorRange[floor]) + .mix(tinycolor(colorRange[ceil]), (pos - floor) * 100) + .toHexString() + + return color +} diff --git a/dashboard/lib/api/api.ts b/dashboard/lib/api/api.ts index af0116e31f3bb..48af21bc4c620 100644 --- a/dashboard/lib/api/api.ts +++ b/dashboard/lib/api/api.ts @@ -26,7 +26,9 @@ export const PREDEFINED_API_ENDPOINTS = [ ] export const DEFAULT_API_ENDPOINT: string = - process.env.NODE_ENV === "production" ? PROD_API_ENDPOINT : MOCK_API_ENDPOINT // EXTERNAL_META_NODE_API_ENDPOINT to debug with RisingWave servers + process.env.NODE_ENV === "production" + ? PROD_API_ENDPOINT + : EXTERNAL_META_NODE_API_ENDPOINT // EXTERNAL_META_NODE_API_ENDPOINT to debug with RisingWave servers export const API_ENDPOINT_KEY = "risingwave.dashboard.api.endpoint" diff --git a/dashboard/lib/api/metric.ts b/dashboard/lib/api/metric.ts index f310e0572170d..23d007bb08360 100644 --- a/dashboard/lib/api/metric.ts +++ b/dashboard/lib/api/metric.ts @@ -156,9 +156,7 @@ export async function fetchEmbeddedBackPressure() { const response: GetBackPressureResponse = await api.get( "/metrics/fragment/embedded_back_pressures" ) - let backPressureInfos: BackPressureInfo[] = - response.backPressureInfos?.map(BackPressureInfo.fromJSON) ?? [] - return backPressureInfos + return response } function calculatePercentile(samples: MetricsSample[], percentile: number) { diff --git a/dashboard/pages/fragment_graph.tsx b/dashboard/pages/fragment_graph.tsx index 2312334e53abe..4b2a39aa5585c 100644 --- a/dashboard/pages/fragment_graph.tsx +++ b/dashboard/pages/fragment_graph.tsx @@ -56,7 +56,7 @@ import { } from "../lib/api/streaming" import { FragmentBox } from "../lib/layout" import { TableFragments, TableFragments_Fragment } from "../proto/gen/meta" -import { BackPressureInfo } from "../proto/gen/monitor_service" +import { BackPressureInfo, FragmentStats } from "../proto/gen/monitor_service" import { Dispatcher, MergeNode, StreamNode } from "../proto/gen/stream_plan" interface DispatcherNode { @@ -329,12 +329,18 @@ export default function Streaming() { // Didn't call `useFetch()` because the `setState` way is special. const [embeddedBackPressureInfo, setEmbeddedBackPressureInfo] = useState() + const [fragmentStats, setFragmentStats] = useState<{ + [key: number]: FragmentStats + }>() + useEffect(() => { if (backPressureDataSource === "Embedded") { - const interval = setInterval(() => { + function refresh() { fetchEmbeddedBackPressure().then( - (newBP) => { - console.log(newBP) + (response) => { + console.log(response) + let newBP = + response.backPressureInfos?.map(BackPressureInfo.fromJSON) ?? [] setEmbeddedBackPressureInfo((prev) => prev ? { @@ -355,13 +361,16 @@ export default function Streaming() { totalDurationNs: 0, } ) + setFragmentStats(response.fragmentStats) }, (e) => { console.error(e) toast(e, "error") } ) - }, INTERVAL_MS) + } + refresh() + const interval = setInterval(refresh, INTERVAL_MS) return () => { clearInterval(interval) } @@ -550,6 +559,7 @@ export default function Streaming() { fragmentDependency={fragmentDependency} planNodeDependencies={planNodeDependencies} backPressures={backPressures} + fragmentStats={fragmentStats} /> )} diff --git a/dashboard/pages/relation_graph.tsx b/dashboard/pages/relation_graph.tsx index a1061ffa23f0d..d44b29708bdc4 100644 --- a/dashboard/pages/relation_graph.tsx +++ b/dashboard/pages/relation_graph.tsx @@ -47,7 +47,7 @@ import { relationIsStreamingJob, } from "../lib/api/streaming" import { RelationPoint } from "../lib/layout" -import { BackPressureInfo } from "../proto/gen/monitor_service" +import { BackPressureInfo, RelationStats } from "../proto/gen/monitor_service" const SIDEBAR_WIDTH = "200px" const INTERVAL_MS = 5000 @@ -133,15 +133,20 @@ export default function StreamingGraph() { // Didn't call `useFetch()` because the `setState` way is special. const [embeddedBackPressureInfo, setEmbeddedBackPressureInfo] = useState() + const [relationStats, setRelationStats] = useState<{ + [key: number]: RelationStats + }>() + useEffect(() => { if (resetEmbeddedBackPressures) { setEmbeddedBackPressureInfo(undefined) toggleResetEmbeddedBackPressures() } if (backPressureDataSource === "Embedded") { - const interval = setInterval(() => { + function refresh() { fetchEmbeddedBackPressure().then( - (newBP) => { + (response) => { + let newBP = response.backPressureInfos setEmbeddedBackPressureInfo((prev) => prev ? { @@ -162,13 +167,16 @@ export default function StreamingGraph() { totalDurationNs: 0, } ) + setRelationStats(response.relationStats) }, (e) => { console.error(e) toast(e, "error") } ) - }, INTERVAL_MS) + } + refresh() + const interval = setInterval(refresh, INTERVAL_MS) return () => { clearInterval(interval) } @@ -308,6 +316,7 @@ export default function StreamingGraph() { selectedId={selectedId?.toString()} setSelectedId={(id) => setSelectedId(parseInt(id))} backPressures={backPressures} + relationStats={relationStats} /> )} diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml index 2ad66049f1a98..a4d5c8760bbe2 100644 --- a/docker/docker-compose-distributed.yml +++ b/docker/docker-compose-distributed.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: compactor-0: <<: *image diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml index 6b32b7357c8a3..ec74ea64a5210 100644 --- a/docker/docker-compose-with-azblob.yml +++ b/docker/docker-compose-with-azblob.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml index 7a25348a2c9ff..d66e60af4330d 100644 --- a/docker/docker-compose-with-gcs.yml +++ b/docker/docker-compose-with-gcs.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml index 9bfd551809328..fece237915243 100644 --- a/docker/docker-compose-with-local-fs.yml +++ b/docker/docker-compose-with-local-fs.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml index e1665b0ee0298..41fbd75c852a5 100644 --- a/docker/docker-compose-with-obs.yml +++ b/docker/docker-compose-with-obs.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml index de9e4c727e760..574a099e0dedc 100644 --- a/docker/docker-compose-with-oss.yml +++ b/docker/docker-compose-with-oss.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml index b00149c0c175b..71672ca6f21a8 100644 --- a/docker/docker-compose-with-s3.yml +++ b/docker/docker-compose-with-s3.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml index ae0b2f31b07b2..0e1e228cfee91 100644 --- a/docker/docker-compose-with-sqlite.yml +++ b/docker/docker-compose-with-sqlite.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index eb0ee6a9b02a2..e7ab7a794e3f5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ --- x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.2} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0} services: risingwave-standalone: <<: *image diff --git a/e2e_test/batch/basic/row_filter.slt.part b/e2e_test/batch/basic/row_filter.slt.part new file mode 100644 index 0000000000000..fdec809d92379 --- /dev/null +++ b/e2e_test/batch/basic/row_filter.slt.part @@ -0,0 +1,65 @@ +statement ok +create table t1(v1 int, v2 int, v3 int); + +statement ok +insert into t1 values(1,1,1),(1,2,1),(1,2,2),(1,3,1),(1,3,2),(1,3,3); + +statement ok +create materialized view mv1 as select * from t1 order by v1 asc, v2 desc, v3 asc; + +statement ok +create materialized view mv2 as select * from t1 order by v1 desc, v2 desc, v3 desc; + +statement ok +create materialized view mv3 as select * from t1 order by v1 asc, v2 asc, v3 asc; + +query III +select * from mv1 where (v1,v2,v3) > (1,3,1) order by v3; +---- +1 3 2 +1 3 3 + +query III +select * from mv2 where (v1,v2,v3) > (1,3,1) order by v3; +---- +1 3 2 +1 3 3 + +query III +select * from mv3 where (v1,v2,v3) > (1,3,1) order by v3; +---- +1 3 2 +1 3 3 + +query III +select * from mv1 where (v1,v2,v3) < (1,3,1) order by v1,v2,v3; +---- +1 1 1 +1 2 1 +1 2 2 + +query III +select * from mv2 where (v1,v2,v3) < (1,3,1) order by v1,v2,v3; +---- +1 1 1 +1 2 1 +1 2 2 + +query III +select * from mv3 where (v1,v2,v3) < (1,3,1) order by v1,v2,v3; +---- +1 1 1 +1 2 1 +1 2 2 + +statement ok +drop materialized view mv3; + +statement ok +drop materialized view mv2; + +statement ok +drop materialized view mv1; + +statement ok +drop table t1; diff --git a/e2e_test/batch/catalog/information_schema.slt b/e2e_test/batch/catalog/information_schema.slt new file mode 100644 index 0000000000000..eed5fbd9d68dc --- /dev/null +++ b/e2e_test/batch/catalog/information_schema.slt @@ -0,0 +1,63 @@ +statement ok +create table t(a int, b bigint, key int primary key); + +statement ok +create view v as select * from t; + +statement ok +create materialized view mv as select * from t; + +query TT +select table_schema, table_name from information_schema.views where table_schema = 'public'; +---- +public v + +query TT +select table_schema, table_name from information_schema.tables where table_schema = 'public' order by table_name; +---- +public mv +public t +public v + +query TTTTTTTT +select constraint_schema, constraint_name, table_schema, table_name from information_schema.table_constraints where table_schema = 'public' order by table_name; +---- +public mv_pkey public mv +public t_pkey public t + +query TT +select schema_name from information_schema.schemata order by schema_name; +---- +information_schema +pg_catalog +public +rw_catalog + +query TTTTTII +select * EXCEPT(constraint_catalog, table_catalog) from information_schema.key_column_usage where table_schema = 'public' order by table_name; +---- +public mv_pkey public mv key 3 NULL +public t_pkey public t key 3 NULL + +query TTTITTT +select table_schema, table_name, column_name, ordinal_position, data_type, udt_schema, udt_name from information_schema.columns where table_schema = 'public' order by table_name, ordinal_position; +---- +public mv a 1 integer pg_catalog int4 +public mv b 2 bigint pg_catalog int8 +public mv key 3 integer pg_catalog int4 +public t a 1 integer pg_catalog int4 +public t b 2 bigint pg_catalog int8 +public t key 3 integer pg_catalog int4 +public v a 1 integer pg_catalog int4 +public v b 2 bigint pg_catalog int8 +public v key 3 integer pg_catalog int4 + + +statement ok +drop materialized view mv; + +statement ok +drop view v; + +statement ok +drop table t; diff --git a/e2e_test/batch/catalog/pg_class.slt.part b/e2e_test/batch/catalog/pg_class.slt.part index f15a2c74fdb40..14aa5c85c2123 100644 --- a/e2e_test/batch/catalog/pg_class.slt.part +++ b/e2e_test/batch/catalog/pg_class.slt.part @@ -2,22 +2,22 @@ query ITIT SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class ORDER BY oid limit 15; ---- 2147478647 columns 1 v -2147478648 schemata 1 v -2147478649 table_constraints 1 v -2147478650 tables 1 v -2147478651 views 1 v -2147478652 pg_am 1 v -2147478653 pg_attrdef 1 v -2147478654 pg_attribute 1 v -2147478655 pg_auth_members 1 v -2147478656 pg_cast 1 r -2147478657 pg_class 1 v -2147478658 pg_collation 1 v -2147478659 pg_constraint 1 r -2147478660 pg_conversion 1 v -2147478661 pg_database 1 v +2147478648 key_column_usage 1 v +2147478649 schemata 1 v +2147478650 table_constraints 1 v +2147478651 tables 1 v +2147478652 views 1 v +2147478653 pg_am 1 v +2147478654 pg_attrdef 1 v +2147478655 pg_attribute 1 v +2147478656 pg_auth_members 1 v +2147478657 pg_cast 1 r +2147478658 pg_class 1 v +2147478659 pg_collation 1 v +2147478660 pg_constraint 1 r +2147478661 pg_conversion 1 v query ITIT SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class WHERE oid = 'pg_namespace'::regclass; ---- -2147478672 pg_namespace 1 v +2147478673 pg_namespace 1 v diff --git a/e2e_test/sink/pg_native_vs_jdbc.slt b/e2e_test/sink/pg_native_vs_jdbc.slt new file mode 100644 index 0000000000000..288d3cdc41053 --- /dev/null +++ b/e2e_test/sink/pg_native_vs_jdbc.slt @@ -0,0 +1,84 @@ +control substitution on + +statement ok +DROP SINK IF EXISTS pg_sink; + +statement ok +DROP SINK IF EXISTS pg_sink_jdbc; + +statement ok +DROP TABLE IF EXISTS datagen_source; + +system ok +psql -c 'DROP TABLE IF EXISTS datagen_source;' + +system ok +psql -c 'DROP TABLE IF EXISTS datagen_source_jdbc;' + +system ok +psql -c 'CREATE TABLE datagen_source (id INT PRIMARY KEY, v1 varchar);' + +system ok +psql -c 'CREATE TABLE datagen_source_jdbc (id INT PRIMARY KEY, v1 varchar);' + +statement ok +CREATE TABLE datagen_source (id INT PRIMARY KEY, v1 varchar); + +statement ok +INSERT INTO datagen_source SELECT key, 'asjdkk2kbdk2uk2ubek2' FROM generate_series(1, 2000000) t(key); + +statement ok +flush; + +statement ok +CREATE SINK pg_sink FROM datagen_source WITH ( + connector='postgres', + host='$PGHOST', + port='$PGPORT', + user='$PGUSER', + password='$PGPASSWORD', + database='$PGDATABASE', + table='datagen_source', + type='upsert', + primary_key='id', +); + +sleep 240s + +system ok +psql --tuples-only -c 'select count(*) from datagen_source;' +---- + 2000000 + + +statement ok +CREATE SINK pg_sink_jdbc FROM datagen_source WITH ( + connector='jdbc', + jdbc.url='jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}?user=${PGUSER}&password=${PGPASSWORD}', + table.name='datagen_source_jdbc', + primary_key='id', + type='upsert' +); + +sleep 240s + +system ok +psql --tuples-only -c 'select count(*) from datagen_source_jdbc;' +---- + 2000000 + + +statement ok +DROP SINK IF EXISTS pg_sink; + +statement ok +DROP SINK IF EXISTS pg_sink_jdbc; + +statement ok +DROP TABLE IF EXISTS datagen_source; + +system ok +psql -c 'DROP TABLE IF EXISTS datagen_source;' + +system ok +psql -c 'DROP TABLE IF EXISTS datagen_source_jdbc;' \ No newline at end of file diff --git a/integration_tests/feature-store/server/Cargo.toml b/integration_tests/feature-store/server/Cargo.toml index 26e300cdf2f6d..8e13d5374d7bf 100644 --- a/integration_tests/feature-store/server/Cargo.toml +++ b/integration_tests/feature-store/server/Cargo.toml @@ -9,16 +9,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -sqlx = { version = "0.8", features = ["runtime-tokio-native-tls", "postgres"] } tokio = { version = "1", features = ["full"] } tonic = "0.11.0" -reqwest = { version = "0.11", features = ["blocking"] } rdkafka = { version = "0.34", features = ["cmake-build"] } -serde_json = "1.0" prost = "0.12" clap = "4.4.6" tokio-postgres = "0.7.10" -tonic-build = "0.11.0" [build-dependencies] tonic-build = "0.11.0" diff --git a/integration_tests/feature-store/simulator/Cargo.toml b/integration_tests/feature-store/simulator/Cargo.toml index e7b53c9047de2..0591ddf5b3137 100644 --- a/integration_tests/feature-store/simulator/Cargo.toml +++ b/integration_tests/feature-store/simulator/Cargo.toml @@ -11,12 +11,10 @@ edition = "2021" [dependencies] tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } tonic = "0.11.0" -reqwest = { version = "0.11" } serde_json = "1.0" serde_derive = "1.0" rand = { workspace = true } clap = "4.4.6" prost = "0.12" -serde = { version = "1", features = ["derive"] } futures = "0.3.28" csv = "1.3.0" diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterHandler.java index 89bd1dd7e6865..604463e578985 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterHandler.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterHandler.java @@ -44,6 +44,6 @@ public static void runJniSinkWriterThread(long requestRxPtr, long responseTxPtr) } catch (Throwable t) { sinkWriterStreamObserver.onError(t); } - LOG.info("end of runJniSinkWriterThread"); + LOG.debug("end of runJniSinkWriterThread"); } } diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java index 1f1c68f54551e..505a22947b655 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java @@ -51,7 +51,7 @@ public void onError(Throwable throwable) { @Override public void onCompleted() { - LOG.info("JniSinkWriterHandler onCompleted"); + LOG.debug("JniSinkWriterHandler onCompleted"); } public boolean isSuccess() { diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java index 53dfe326fbd9d..55e5befa62ea6 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java @@ -187,7 +187,7 @@ public void onError(Throwable throwable) { @Override public void onCompleted() { - LOG.info("sink writer completed"); + LOG.debug("sink writer completed"); cleanup(); responseObserver.onCompleted(); } diff --git a/proto/monitor_service.proto b/proto/monitor_service.proto index 772d43e5613c2..08fc97bb73f9b 100644 --- a/proto/monitor_service.proto +++ b/proto/monitor_service.proto @@ -61,8 +61,20 @@ message BackPressureInfo { double value = 4; } +message FragmentStats { + uint32 actor_count = 2; + uint64 current_epoch = 3; +} + +message RelationStats { + uint32 actor_count = 2; + uint64 current_epoch = 3; +} + message GetBackPressureResponse { repeated BackPressureInfo back_pressure_infos = 1; + map fragment_stats = 2; + map relation_stats = 3; } message TieredCacheTracingRequest { diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto index 8aabadfd2b74d..3963e3d03e4c3 100644 --- a/proto/stream_plan.proto +++ b/proto/stream_plan.proto @@ -627,6 +627,8 @@ message StreamScanNode { // The state table used by ArrangementBackfill to replicate upstream mview's state table. // Used iff `ChainType::ArrangementBackfill`. catalog.Table arrangement_table = 10; + + optional uint64 snapshot_backfill_epoch = 11; } // Config options for CDC backfill diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml index c501f2df4962e..56491055cfacd 100644 --- a/src/batch/Cargo.toml +++ b/src/batch/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" async-recursion = "1" @@ -62,11 +56,9 @@ twox-hash = "2" workspace-hack = { path = "../workspace-hack" } [dev-dependencies] -criterion = { workspace = true, features = ["async_tokio", "async"] } rand = { workspace = true } risingwave_hummock_sdk = { workspace = true } tempfile = "3" -tikv-jemallocator = { workspace = true } [lints] workspace = true diff --git a/src/batch/executors/Cargo.toml b/src/batch/executors/Cargo.toml index 927c026e83d8e..736c54f8a5b0e 100644 --- a/src/batch/executors/Cargo.toml +++ b/src/batch/executors/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" assert_matches = "1" diff --git a/src/batch/executors/src/executor/row_seq_scan.rs b/src/batch/executors/src/executor/row_seq_scan.rs index 445b1fa8038d7..a9efb465df187 100644 --- a/src/batch/executors/src/executor/row_seq_scan.rs +++ b/src/batch/executors/src/executor/row_seq_scan.rs @@ -434,6 +434,7 @@ impl RowSeqScanExecutor { next_col_bounds, } = scan_range; + // The len of a valid pk_prefix should be less than or equal pk's num. let order_type = table.pk_serializer().get_order_types()[pk_prefix.len()]; let (start_bound, end_bound) = if order_type.is_ascending() { (next_col_bounds.0, next_col_bounds.1) diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 5ddc6112bd4ab..097f9fd881a48 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -21,7 +21,6 @@ normal = ["workspace-hack", "workspace-config", "task_stats_alloc"] [dependencies] clap = { workspace = true } -prometheus = { version = "0.13" } risingwave_batch_executors = { workspace = true } risingwave_common = { workspace = true } risingwave_compactor = { workspace = true } diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml index 2269ca1010481..0e74a20a4a10e 100644 --- a/src/cmd_all/Cargo.toml +++ b/src/cmd_all/Cargo.toml @@ -19,18 +19,16 @@ js-udf = ["risingwave_expr_impl/js-udf"] python-udf = ["risingwave_expr_impl/python-udf"] [package.metadata.cargo-machete] -ignored = ["workspace-hack", "workspace-config", "task_stats_alloc"] +ignored = ["workspace-hack", "workspace-config", "task_stats_alloc", "tikv-jemallocator"] [package.metadata.cargo-udeps.ignore] -ignored = ["workspace-hack", "workspace-config", "task_stats_alloc"] +ignored = ["workspace-hack", "workspace-config", "task_stats_alloc", "tikv-jemallocator"] [dependencies] -anyhow = "1" clap = { workspace = true } console = "0.15" const-str = "0.5" home = "0.5" -prometheus = { version = "0.13" } risingwave_batch_executors = { workspace = true } risingwave_cmd = { workspace = true } risingwave_common = { workspace = true } diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 8a554137dccdd..1fbf8e312185d 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] ahash = "0.8" anyhow = "1" @@ -48,12 +42,11 @@ ethnum = { version = "1", features = ["serde"] } fixedbitset = "0.5" foyer = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } -governor = { version = "0.6", default-features = false, features = ["std"] } +governor = { workspace = true } hashbrown = "0.14" hex = "0.4.3" http = "1" humantime = "2.1" -hytra = { workspace = true } itertools = { workspace = true } itoa = "1.0" jsonbb = { workspace = true } diff --git a/src/common/common_service/Cargo.toml b/src/common/common_service/Cargo.toml index 37775ff04a82b..38dd8f4534477 100644 --- a/src/common/common_service/Cargo.toml +++ b/src/common/common_service/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] async-trait = "0.1" axum = { workspace = true } diff --git a/src/common/estimate_size/Cargo.toml b/src/common/estimate_size/Cargo.toml index 77e4203f9c7cb..c6447ff974216 100644 --- a/src/common/estimate_size/Cargo.toml +++ b/src/common/estimate_size/Cargo.toml @@ -8,19 +8,12 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] bytes = "1" educe = "0.6" ethnum = { version = "1", features = ["serde"] } fixedbitset = "0.5" jsonbb = { workspace = true } -lru = { workspace = true } risingwave_common_proc_macro = { workspace = true } rust_decimal = "1" serde_json = "1" diff --git a/src/common/heap_profiling/Cargo.toml b/src/common/heap_profiling/Cargo.toml index 706f801fe4762..edb0d11e2fd0b 100644 --- a/src/common/heap_profiling/Cargo.toml +++ b/src/common/heap_profiling/Cargo.toml @@ -8,14 +8,7 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] -anyhow = "1" chrono = { version = "0.4", default-features = false, features = [ "clock", "std", diff --git a/src/common/metrics/Cargo.toml b/src/common/metrics/Cargo.toml index 78562624c6935..b499508f177d9 100644 --- a/src/common/metrics/Cargo.toml +++ b/src/common/metrics/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] auto_impl = "1" bytes = "1" diff --git a/src/common/metrics/src/gauge_ext.rs b/src/common/metrics/src/gauge_ext.rs index b76f2ac7531d0..aa5471ec87171 100644 --- a/src/common/metrics/src/gauge_ext.rs +++ b/src/common/metrics/src/gauge_ext.rs @@ -12,8 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use prometheus::core::{AtomicU64, GenericGauge}; use prometheus::IntGauge; +/// The integer version of [`prometheus::Gauge`]. Provides better performance if metric values are +/// all unsigned integers. +pub type UintGauge = GenericGauge; + #[easy_ext::ext(IntGaugeExt)] impl IntGauge { /// Increment the gauge, and return a guard that will decrement the gauge when dropped. @@ -39,3 +44,29 @@ impl IntGauge { Guard::create(self) } } + +#[easy_ext::ext(UintGaugeExt)] +impl UintGauge { + /// Increment the gauge, and return a guard that will decrement the gauge when dropped. + #[must_use] + pub fn inc_guard(&self) -> impl Drop + '_ { + struct Guard<'a> { + gauge: &'a UintGauge, + } + + impl<'a> Guard<'a> { + fn create(gauge: &'a UintGauge) -> Self { + gauge.inc(); + Self { gauge } + } + } + + impl Drop for Guard<'_> { + fn drop(&mut self) { + self.gauge.dec(); + } + } + + Guard::create(self) + } +} diff --git a/src/common/metrics/src/guarded_metrics.rs b/src/common/metrics/src/guarded_metrics.rs index 8e953e28ac948..809e205db0ec2 100644 --- a/src/common/metrics/src/guarded_metrics.rs +++ b/src/common/metrics/src/guarded_metrics.rs @@ -86,7 +86,7 @@ macro_rules! register_guarded_int_gauge_vec_with_registry { #[macro_export] macro_rules! register_guarded_uint_gauge_vec_with_registry { ($NAME:expr, $HELP:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{ - let inner = prometheus::core::GenericGaugeVec::::new( + let inner = prometheus::core::GenericGaugeVec::::new( prometheus::opts!($NAME, $HELP), $LABELS_NAMES, ); @@ -142,6 +142,8 @@ mod tait { } pub use tait::*; +use crate::UintGauge; + pub type LabelGuardedHistogramVec = LabelGuardedMetricVec; pub type LabelGuardedIntCounterVec = LabelGuardedMetricVec, N>; @@ -155,6 +157,7 @@ pub type LabelGuardedGaugeVec = pub type LabelGuardedHistogram = LabelGuardedMetric; pub type LabelGuardedIntCounter = LabelGuardedMetric; pub type LabelGuardedIntGauge = LabelGuardedMetric; +pub type LabelGuardedUintGauge = LabelGuardedMetric; pub type LabelGuardedGauge = LabelGuardedMetric; pub type LabelGuardedLocalHistogram = LabelGuardedMetric; diff --git a/src/common/proc_macro/Cargo.toml b/src/common/proc_macro/Cargo.toml index 2219dc2efaba4..d44951a4aec56 100644 --- a/src/common/proc_macro/Cargo.toml +++ b/src/common/proc_macro/Cargo.toml @@ -10,11 +10,9 @@ repository = { workspace = true } [lib] proc-macro = true -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] + + [dependencies] proc-macro-error = "1.0" diff --git a/src/common/secret/Cargo.toml b/src/common/secret/Cargo.toml index 6d501594acbde..ebd042b23a9b0 100644 --- a/src/common/secret/Cargo.toml +++ b/src/common/secret/Cargo.toml @@ -7,11 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] [dependencies] aes-gcm = "0.10" diff --git a/src/common/src/util/scan_range.rs b/src/common/src/util/scan_range.rs index b45db071be3cc..5c56550eed279 100644 --- a/src/common/src/util/scan_range.rs +++ b/src/common/src/util/scan_range.rs @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::Ordering; use std::ops::{Bound, RangeBounds}; +use itertools::Itertools; use paste::paste; use risingwave_pb::batch_plan::scan_range::Bound as PbBound; use risingwave_pb::batch_plan::ScanRange as PbScanRange; +use super::sort_util::{cmp_rows, OrderType}; use crate::hash::table_distribution::TableDistribution; use crate::hash::VirtualNode; use crate::types::{Datum, ScalarImpl}; @@ -98,6 +101,147 @@ impl ScanRange { range: full_range(), } } + + pub fn convert_to_range(&self) -> (Bound>, Bound>) { + fn handle_bound(eq_conds: &Vec, bound: &Bound>) -> Bound> { + match bound { + Bound::Included(literal) => { + let mut prefix = eq_conds.clone(); + prefix.extend_from_slice(literal); + Bound::Included(prefix) + } + Bound::Excluded(literal) => { + let mut prefix = eq_conds.clone(); + prefix.extend_from_slice(literal); + Bound::Excluded(prefix) + } + Bound::Unbounded => { + if eq_conds.is_empty() { + Bound::Unbounded + } else { + Bound::Included(eq_conds.clone()) + } + } + } + } + + let new_left = handle_bound(&self.eq_conds, &self.range.0); + let new_right = handle_bound(&self.eq_conds, &self.range.1); + (new_left, new_right) + } + + pub fn is_overlap(left: &ScanRange, right: &ScanRange, order_types: &[OrderType]) -> bool { + let range_left = left.convert_to_range(); + let range_right = right.convert_to_range(); + Self::range_overlap_check(range_left, range_right, order_types) + } + + fn range_overlap_check( + left: (Bound>, Bound>), + right: (Bound>, Bound>), + order_types: &[OrderType], + ) -> bool { + let (left_start, left_end) = &left; + let (right_start, right_end) = &right; + + let left_start_vec = match &left_start { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + let right_start_vec = match &right_start { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + + if left_start_vec.is_empty() && right_start_vec.is_empty() { + return true; + } + + let order_types = if order_types.iter().all(|o| o.is_ascending()) { + order_types + } else { + // reverse order types to ascending + &order_types + .iter() + .cloned() + .map(|o| if o.is_descending() { o.reverse() } else { o }) + .collect_vec() + }; + + // Unbounded is always less than any other bound + if left_start_vec.is_empty() { + // pass + } else if right_start_vec.is_empty() { + return Self::range_overlap_check(right, left, order_types); + } else { + assert!(!left_start_vec.is_empty()); + assert!(!right_start_vec.is_empty()); + let cmp_column_len = left_start_vec.len().min(right_start_vec.len()); + let cmp_start = cmp_rows( + &left_start_vec[0..cmp_column_len], + &right_start_vec[0..cmp_column_len], + &order_types[0..cmp_column_len], + ); + + let right_start_before_left_start = cmp_start.is_gt(); + + if right_start_before_left_start { + return Self::range_overlap_check(right, left, order_types); + } + + if cmp_start == Ordering::Equal + && let (Bound::Included(_), Bound::Included(_)) = (left_start, right_start) + { + return true; + } + } + + let left_end_vec = match &left_end { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + let right_end_vec = match &right_end { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + + if left_end_vec.is_empty() && right_end_vec.is_empty() { + return true; + } + + if left_end_vec.is_empty() { + true + } else { + // cmp left_end and right_start + assert!(!left_end_vec.is_empty()); + assert!(!right_start_vec.is_empty()); + + let cmp_column_len = left_end_vec.len().min(right_start_vec.len()); + let cmp_end = cmp_rows( + &left_end_vec[0..cmp_column_len], + &right_start_vec[0..cmp_column_len], + &order_types[0..cmp_column_len], + ); + + match cmp_end { + Ordering::Equal => { + if let (Bound::Included(_), Bound::Included(_)) = (left_end, right_start) { + return true; + } + } + + Ordering::Greater => { + return true; + } + + Ordering::Less => { + return false; + } + } + + false + } + } } pub const fn full_range() -> (Bound, Bound) { @@ -221,4 +365,465 @@ mod tests { assert_eq!(scan_range.try_compute_vnode(&dist), Some(vnode)); } + + #[test] + fn test_convert_to_range() { + { + // test empty eq_conds + let scan_range = ScanRange { + eq_conds: vec![], + range: ( + Bound::Included(vec![Some(ScalarImpl::from(1))]), + Bound::Included(vec![Some(ScalarImpl::from(2))]), + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!(left, Bound::Included(vec![Some(ScalarImpl::from(1))])); + assert_eq!(right, Bound::Included(vec![Some(ScalarImpl::from(2))])); + } + + { + // test exclude bound with empty eq_conds + let scan_range = ScanRange { + eq_conds: vec![], + range: ( + Bound::Excluded(vec![Some(ScalarImpl::from(1))]), + Bound::Excluded(vec![Some(ScalarImpl::from(2))]), + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!(left, Bound::Excluded(vec![Some(ScalarImpl::from(1))])); + assert_eq!(right, Bound::Excluded(vec![Some(ScalarImpl::from(2))])); + } + + { + // test include bound with empty eq_conds + let scan_range = ScanRange { + eq_conds: vec![], + range: ( + Bound::Included(vec![Some(ScalarImpl::from(1))]), + Bound::Unbounded, + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!(left, Bound::Included(vec![Some(ScalarImpl::from(1))])); + assert_eq!(right, Bound::Unbounded); + } + + { + // test exclude bound with non-empty eq_conds + let scan_range = ScanRange { + eq_conds: vec![Some(ScalarImpl::from(1))], + range: ( + Bound::Excluded(vec![Some(ScalarImpl::from(2))]), + Bound::Excluded(vec![Some(ScalarImpl::from(3))]), + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!( + left, + Bound::Excluded(vec![Some(ScalarImpl::from(1)), Some(ScalarImpl::from(2))]) + ); + assert_eq!( + right, + Bound::Excluded(vec![Some(ScalarImpl::from(1)), Some(ScalarImpl::from(3))]) + ); + } + + { + // test include bound with non-empty eq_conds + let scan_range = ScanRange { + eq_conds: vec![Some(ScalarImpl::from(1))], + range: ( + Bound::Included(vec![Some(ScalarImpl::from(2))]), + Bound::Included(vec![Some(ScalarImpl::from(3))]), + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!( + left, + Bound::Included(vec![Some(ScalarImpl::from(1)), Some(ScalarImpl::from(2))]) + ); + assert_eq!( + right, + Bound::Included(vec![Some(ScalarImpl::from(1)), Some(ScalarImpl::from(3))]) + ); + } + + { + let scan_range = ScanRange { + eq_conds: vec![Some(ScalarImpl::from(1))], + range: ( + Bound::Included(vec![Some(ScalarImpl::from(2))]), + Bound::Unbounded, + ), + }; + + let (left, right) = scan_range.convert_to_range(); + assert_eq!( + left, + Bound::Included(vec![Some(ScalarImpl::from(1)), Some(ScalarImpl::from(2))]) + ); + assert_eq!(right, Bound::Included(vec![Some(ScalarImpl::from(1))])); + } + } + + #[test] + fn test_range_overlap_check() { + let order_types = vec![OrderType::ascending()]; + + // (Included, Included) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Excluded, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Excluded, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Excluded) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Excluded) vs (Included, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Excluded) vs (Excluded, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Excluded) vs (Excluded, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Included) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Included) vs (Included, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Included) vs (Excluded, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Included) vs (Excluded, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Excluded) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Excluded) vs (Included, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Excluded) vs (Excluded, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Excluded, Excluded) vs (Excluded, Excluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(5))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3)), Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7)), Some(ScalarImpl::Int32(7))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(5))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Included) + assert!(!ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(2))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Included) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(3))]), + Bound::Included(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Excluded, Encluded) + assert!(!ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Excluded(vec![Some(ScalarImpl::Int32(3))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + // (Included, Included) vs (Included, Encluded) + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Excluded(vec![Some(ScalarImpl::Int32(7))]) + ), + &order_types + )); + + assert!(!ScanRange::range_overlap_check( + ( + Bound::Unbounded, + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(5))]), + Bound::Unbounded, + ), + &order_types + )); + + assert!(ScanRange::range_overlap_check( + ( + Bound::Unbounded, + Bound::Included(vec![Some(ScalarImpl::Int32(10))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(5))]), + Bound::Unbounded, + ), + &order_types + )); + + assert!(ScanRange::range_overlap_check( + (Bound::Unbounded, Bound::Unbounded,), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(5))]), + Bound::Unbounded, + ), + &order_types + )); + + assert!(ScanRange::range_overlap_check( + (Bound::Unbounded, Bound::Unbounded), + (Bound::Unbounded, Bound::Unbounded), + &order_types + )); + + assert!(!ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Included(vec![Some(ScalarImpl::Int32(5))]), + Bound::Unbounded, + ), + &order_types + )); + + assert!(ScanRange::range_overlap_check( + ( + Bound::Included(vec![Some(ScalarImpl::Int32(1))]), + Bound::Included(vec![Some(ScalarImpl::Int32(3))]) + ), + ( + Bound::Unbounded, + Bound::Included(vec![Some(ScalarImpl::Int32(5))]), + ), + &order_types + )); + } } diff --git a/src/common/telemetry_event/Cargo.toml b/src/common/telemetry_event/Cargo.toml index 6007f1cdc3314..ab3826b9504b0 100644 --- a/src/common/telemetry_event/Cargo.toml +++ b/src/common/telemetry_event/Cargo.toml @@ -7,18 +7,11 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] jsonbb = { workspace = true } prost = { workspace = true } reqwest = { version = "0.12.2", features = ["json"] } risingwave_pb = { workspace = true } -thiserror = "1" thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", @@ -29,4 +22,3 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ "signal", ] } tracing = "0.1" -uuid = { version = "1", features = ["v4"] } diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index ef66755e3e5be..d6a14aa04cd9e 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" async-trait = "0.1" @@ -24,7 +18,6 @@ foyer = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } futures-async-stream = { workspace = true } http = "1" -hyper = "1" itertools = { workspace = true } maplit = "1.0.2" pprof = { version = "0.14", features = ["flamegraph"] } @@ -60,7 +53,6 @@ tokio-stream = { workspace = true } tonic = { workspace = true } tower = { version = "0.5", features = ["util", "load-shed"] } tracing = "0.1" -uuid = { version = "1.11.0", features = ["v4"] } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../workspace-hack" } diff --git a/src/compute/src/rpc/service/monitor_service.rs b/src/compute/src/rpc/service/monitor_service.rs index 2671fd533262f..cf0b76919fb59 100644 --- a/src/compute/src/rpc/service/monitor_service.rs +++ b/src/compute/src/rpc/service/monitor_service.rs @@ -21,16 +21,18 @@ use std::time::Duration; use foyer::{HybridCache, TracingOptions}; use itertools::Itertools; use prometheus::core::Collector; +use prometheus::proto::Metric; use risingwave_common::config::{MetricLevel, ServerConfig}; use risingwave_common_heap_profiling::{AUTO_DUMP_SUFFIX, COLLAPSED_SUFFIX, MANUALLY_DUMP_SUFFIX}; use risingwave_hummock_sdk::HummockSstableObjectId; use risingwave_jni_core::jvm_runtime::dump_jvm_stack_traces; use risingwave_pb::monitor_service::monitor_service_server::MonitorService; use risingwave_pb::monitor_service::{ - AnalyzeHeapRequest, AnalyzeHeapResponse, BackPressureInfo, GetBackPressureRequest, - GetBackPressureResponse, HeapProfilingRequest, HeapProfilingResponse, ListHeapProfilingRequest, - ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, StackTraceRequest, - StackTraceResponse, TieredCacheTracingRequest, TieredCacheTracingResponse, + AnalyzeHeapRequest, AnalyzeHeapResponse, BackPressureInfo, FragmentStats, + GetBackPressureRequest, GetBackPressureResponse, HeapProfilingRequest, HeapProfilingResponse, + ListHeapProfilingRequest, ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, + RelationStats, StackTraceRequest, StackTraceResponse, TieredCacheTracingRequest, + TieredCacheTracingResponse, }; use risingwave_rpc_client::error::ToTonicStatus; use risingwave_storage::hummock::compactor::await_tree_key::Compaction; @@ -309,19 +311,77 @@ impl MonitorService for MonitorServiceImpl { let actor_count: HashMap<_, _> = actor_count .iter() - .filter_map(|m| { - let fragment_id = m - .get_label() - .iter() - .find(|lp| lp.get_name() == "fragment_id")? - .get_value() - .parse::() - .unwrap(); + .map(|m| { + let fragment_id = get_label(m, "fragment_id").unwrap(); let count = m.get_gauge().get_value() as u32; - Some((fragment_id, count)) + (fragment_id, count) }) .collect(); + let mut fragment_stats: HashMap = HashMap::new(); + for (&fragment_id, &actor_count) in &actor_count { + fragment_stats.insert( + fragment_id, + FragmentStats { + actor_count, + current_epoch: 0, + }, + ); + } + + let actor_current_epoch = metrics + .actor_current_epoch + .collect() + .into_iter() + .next() + .unwrap() + .take_metric(); + for m in &actor_current_epoch { + let fragment_id = get_label(m, "fragment_id").unwrap(); + let epoch = m.get_gauge().get_value() as u64; + if let Some(s) = fragment_stats.get_mut(&fragment_id) { + s.current_epoch = if s.current_epoch == 0 { + epoch + } else { + u64::min(s.current_epoch, epoch) + } + } else { + warn!( + fragment_id = fragment_id, + "Miss corresponding actor count metrics" + ); + } + } + + let mut relation_stats: HashMap = HashMap::new(); + let mview_current_epoch = metrics + .materialize_current_epoch + .collect() + .into_iter() + .next() + .unwrap() + .take_metric(); + for m in &mview_current_epoch { + let table_id = get_label(m, "table_id").unwrap(); + let epoch = m.get_gauge().get_value() as u64; + if let Some(s) = relation_stats.get_mut(&table_id) { + s.current_epoch = if s.current_epoch == 0 { + epoch + } else { + u64::min(s.current_epoch, epoch) + }; + s.actor_count += 1; + } else { + relation_stats.insert( + table_id, + RelationStats { + actor_count: 1, + current_epoch: epoch, + }, + ); + } + } + let mut back_pressure_infos: HashMap<_, BackPressureInfo> = HashMap::new(); for label_pairs in actor_output_buffer_blocking_duration_ns { @@ -360,6 +420,8 @@ impl MonitorService for MonitorServiceImpl { Ok(Response::new(GetBackPressureResponse { back_pressure_infos: back_pressure_infos.into_values().collect(), + fragment_stats, + relation_stats, })) } @@ -436,6 +498,16 @@ impl MonitorService for MonitorServiceImpl { } } +fn get_label(metric: &Metric, label: &str) -> Option { + metric + .get_label() + .iter() + .find(|lp| lp.get_name() == label)? + .get_value() + .parse::() + .ok() +} + pub use grpc_middleware::*; pub mod grpc_middleware { diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 3f12926c9edac..b1ab8b9a8a72c 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" apache-avro = { workspace = true } @@ -61,12 +55,7 @@ google-cloud-bigquery = { version = "0.13.0", features = ["auth"] } google-cloud-gax = "0.19.0" google-cloud-googleapis = { version = "0.15", features = ["pubsub", "bigquery"] } google-cloud-pubsub = "0.29" -governor = { version = "0.6", default-features = false, features = [ - "std", - "dashmap", - "jitter", -] } -http = "0.2" +governor = { workspace = true } iceberg = { workspace = true } iceberg-catalog-glue = { workspace = true } iceberg-catalog-rest = { workspace = true } @@ -74,8 +63,6 @@ icelake = { workspace = true } indexmap = { version = "2.7.0", features = ["serde"] } itertools = { workspace = true } jni = { version = "0.21.1", features = ["invocation"] } -jsonbb = { workspace = true } -jsonwebtoken = "9.2.0" maplit = "1.0.2" moka = { version = "0.12.8", features = ["future"] } mongodb = { version = "2.8.2", features = ["tokio-runtime"] } @@ -125,7 +112,6 @@ risingwave_common = { workspace = true } risingwave_common_estimate_size = { workspace = true } risingwave_connector_codec = { workspace = true } risingwave_jni_core = { workspace = true } -risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } rumqttc = { version = "0.24.0", features = ["url"] } diff --git a/src/connector/codec/Cargo.toml b/src/connector/codec/Cargo.toml index 9038c928dc3da..50b3a53887404 100644 --- a/src/connector/codec/Cargo.toml +++ b/src/connector/codec/Cargo.toml @@ -8,12 +8,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" apache-avro = { workspace = true } diff --git a/src/connector/src/sink/mongodb.rs b/src/connector/src/sink/mongodb.rs index 244a1bb70db8d..44d116cedf093 100644 --- a/src/connector/src/sink/mongodb.rs +++ b/src/connector/src/sink/mongodb.rs @@ -81,6 +81,12 @@ mod send_bulk_write_command_future { ))) })?; + if let Ok(ok) = result.get_i32("ok") + && ok != 1 + { + return Err(SinkError::Mongodb(anyhow!("bulk write write errors"))); + } + if let Ok(write_errors) = result.get_array("writeErrors") { return Err(SinkError::Mongodb(anyhow!( "bulk write respond with write errors: {:?}", @@ -88,15 +94,10 @@ mod send_bulk_write_command_future { ))); } - let n = result.get_i32("n").map_err(|err| { - SinkError::Mongodb( - anyhow!(err).context("can't extract field n from bulk write response"), - ) - })?; - if n < 1 { + if let Ok(write_concern_error) = result.get_array("writeConcernError") { return Err(SinkError::Mongodb(anyhow!( - "bulk write respond with an abnormal state, n = {}", - n + "bulk write respond with write errors: {:?}", + write_concern_error, ))); } diff --git a/src/connector/src/sink/postgres.rs b/src/connector/src/sink/postgres.rs index 02abe81f020e1..f077e9ac42d9d 100644 --- a/src/connector/src/sink/postgres.rs +++ b/src/connector/src/sink/postgres.rs @@ -13,48 +13,30 @@ // limitations under the License. use std::collections::{BTreeMap, HashSet}; -use std::sync::Arc; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use async_trait::async_trait; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::bitmap::Bitmap; +use risingwave_common::bail; use risingwave_common::catalog::Schema; use risingwave_common::row::{Row, RowExt}; use serde_derive::Deserialize; use serde_with::{serde_as, DisplayFromStr}; use simd_json::prelude::ArrayTrait; use thiserror_ext::AsReport; -use tokio_postgres::Statement; +use tokio_postgres::types::Type as PgType; use super::{ - SinkError, SinkWriterMetrics, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT, + LogSinker, SinkError, SinkLogReader, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT, }; use crate::connector_common::{create_pg_client, PostgresExternalTable, SslMode}; use crate::parser::scalar_adapter::{validate_pg_type_to_rw_type, ScalarAdapter}; -use crate::sink::writer::{LogSinkerOf, SinkWriter, SinkWriterExt}; +use crate::sink::log_store::{LogStoreReadItem, TruncateOffset}; use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkParam, SinkWriterParam}; pub const POSTGRES_SINK: &str = "postgres"; -macro_rules! rw_row_to_pg_values { - ($row:expr, $statement:expr) => { - $row.iter().enumerate().map(|(i, d)| { - d.and_then(|d| { - let ty = &$statement.params()[i]; - match ScalarAdapter::from_scalar(d, ty) { - Ok(scalar) => Some(scalar), - Err(e) => { - tracing::error!(error=%e.as_report(), scalar=?d, "Failed to convert scalar to pg value"); - None - } - } - }) - }) - }; -} - #[serde_as] #[derive(Clone, Debug, Deserialize)] pub struct PostgresConfig { @@ -143,7 +125,7 @@ impl TryFrom for PostgresSink { impl Sink for PostgresSink { type Coordinator = DummySinkCommitCoordinator; - type LogSinker = LogSinkerOf; + type LogSinker = PostgresSinkWriter; const SINK_NAME: &'static str = POSTGRES_SINK; @@ -239,40 +221,78 @@ impl Sink for PostgresSink { Ok(()) } - async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result { - Ok(PostgresSinkWriter::new( + async fn new_log_sinker(&self, _writer_param: SinkWriterParam) -> Result { + PostgresSinkWriter::new( self.config.clone(), self.schema.clone(), self.pk_indices.clone(), self.is_append_only, ) - .await? - .into_log_sinker(SinkWriterMetrics::new(&writer_param))) + .await } } -struct Buffer { - buffer: Vec, - size: usize, +struct ParameterBuffer<'a> { + /// A set of parameters to be inserted/deleted. + /// Each set is a flattened 2d-array. + parameters: Vec>>, + /// the column dimension (fixed). + column_length: usize, + /// schema types to serialize into `ScalarAdapter` + schema_types: &'a [PgType], + /// estimated number of parameters that can be sent in a single query. + estimated_parameter_size: usize, + /// current parameter buffer to be filled. + current_parameter_buffer: Vec>, } -impl Buffer { - fn new() -> Self { +impl<'a> ParameterBuffer<'a> { + /// The maximum number of parameters that can be sent in a single query. + /// See: + /// and + const MAX_PARAMETERS: usize = 32768; + + /// `flattened_chunk_size` is the number of datums in a single chunk. + fn new(schema_types: &'a [PgType], flattened_chunk_size: usize) -> Self { + let estimated_parameter_size = usize::min(Self::MAX_PARAMETERS, flattened_chunk_size); Self { - buffer: Vec::new(), - size: 0, + parameters: vec![], + column_length: schema_types.len(), + schema_types, + estimated_parameter_size, + current_parameter_buffer: Vec::with_capacity(estimated_parameter_size), } } - fn push(&mut self, chunk: StreamChunk) -> usize { - self.size += chunk.cardinality(); - self.buffer.push(chunk); - self.size + fn add_row(&mut self, row: impl Row) { + if self.current_parameter_buffer.len() + self.column_length >= Self::MAX_PARAMETERS { + self.new_buffer(); + } + for (i, datum_ref) in row.iter().enumerate() { + let pg_datum = datum_ref.map(|s| { + let ty = &self.schema_types[i]; + match ScalarAdapter::from_scalar(s, ty) { + Ok(scalar) => Some(scalar), + Err(e) => { + tracing::error!(error=%e.as_report(), scalar=?s, "Failed to convert scalar to pg value"); + None + } + } + }); + self.current_parameter_buffer.push(pg_datum.flatten()); + } + } + + fn new_buffer(&mut self) { + let filled_buffer = std::mem::replace( + &mut self.current_parameter_buffer, + Vec::with_capacity(self.estimated_parameter_size), + ); + self.parameters.push(filled_buffer); } - fn drain(&mut self) -> Vec { - self.size = 0; - std::mem::take(&mut self.buffer) + fn into_parts(self) -> (Vec>>, Vec>) { + (self.parameters, self.current_parameter_buffer) } } @@ -281,10 +301,8 @@ pub struct PostgresSinkWriter { pk_indices: Vec, is_append_only: bool, client: tokio_postgres::Client, - buffer: Buffer, - insert_statement: Statement, - delete_statement: Option, - upsert_statement: Option, + schema_types: Vec, + schema: Schema, } impl PostgresSinkWriter { @@ -338,201 +356,227 @@ impl PostgresSinkWriter { schema_types }; - let insert_statement = { - let insert_sql = create_insert_sql(&schema, &config.table); - client - .prepare_typed(&insert_sql, &schema_types) - .await - .context("Failed to prepare insert statement")? - }; - - let delete_statement = if is_append_only { - None - } else { - let delete_types = pk_indices - .iter() - .map(|i| schema_types[*i].clone()) - .collect_vec(); - let delete_sql = create_delete_sql(&schema, &config.table, &pk_indices); - Some( - client - .prepare_typed(&delete_sql, &delete_types) - .await - .context("Failed to prepare delete statement")?, - ) - }; - - let upsert_statement = if is_append_only { - None - } else { - let upsert_sql = create_upsert_sql(&schema, &config.table, &pk_indices); - Some( - client - .prepare_typed(&upsert_sql, &schema_types) - .await - .context("Failed to prepare upsert statement")?, - ) - }; - let writer = Self { config, pk_indices, is_append_only, client, - buffer: Buffer::new(), - insert_statement, - delete_statement, - upsert_statement, + schema_types, + schema, }; Ok(writer) } - async fn flush(&mut self) -> Result<()> { + async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> { + // https://www.postgresql.org/docs/current/limits.html + // We have a limit of 65,535 parameters in a single query, as restricted by the PostgreSQL protocol. if self.is_append_only { - for chunk in self.buffer.drain() { - for (op, row) in chunk.rows() { - match op { - Op::Insert => { - self.client - .execute_raw( - &self.insert_statement, - rw_row_to_pg_values!(row, self.insert_statement), - ) - .await?; - } - Op::UpdateInsert | Op::Delete | Op::UpdateDelete => { - debug_assert!(!self.is_append_only); - } - } - } - } + self.write_batch_append_only(chunk).await } else { - let mut unmatched_update_insert = 0; - for chunk in self.buffer.drain() { - for (op, row) in chunk.rows() { - match op { - Op::Insert => { - self.client - .execute_raw( - &self.insert_statement, - rw_row_to_pg_values!(row, self.insert_statement), - ) - .await?; - } - Op::UpdateInsert => { - unmatched_update_insert += 1; - self.client - .execute_raw( - self.upsert_statement.as_ref().unwrap(), - rw_row_to_pg_values!( - row, - self.upsert_statement.as_ref().unwrap() - ), - ) - .await?; - } - Op::Delete => { - self.client - .execute_raw( - self.delete_statement.as_ref().unwrap(), - rw_row_to_pg_values!( - row.project(&self.pk_indices), - self.delete_statement.as_ref().unwrap() - ), - ) - .await?; - } - Op::UpdateDelete => { - unmatched_update_insert -= 1; - } - } + self.write_batch_non_append_only(chunk).await + } + } + + async fn write_batch_append_only(&mut self, chunk: StreamChunk) -> Result<()> { + let mut transaction = self.client.transaction().await?; + // 1d flattened array of parameters to be inserted. + let mut parameter_buffer = ParameterBuffer::new( + &self.schema_types, + chunk.cardinality() * chunk.data_types().len(), + ); + for (op, row) in chunk.rows() { + match op { + Op::Insert => { + parameter_buffer.add_row(row); + } + Op::UpdateInsert | Op::Delete | Op::UpdateDelete => { + bail!("append-only sink should not receive update insert, update delete and delete operations") } } - assert_eq!(unmatched_update_insert, 0); } + let (parameters, remaining) = parameter_buffer.into_parts(); + Self::execute_parameter( + Op::Insert, + &mut transaction, + &self.schema, + &self.config.table, + &self.pk_indices, + parameters, + remaining, + ) + .await?; + transaction.commit().await?; Ok(()) } -} - -#[async_trait] -impl SinkWriter for PostgresSinkWriter { - async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> { - Ok(()) - } - async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> { - let cardinality = self.buffer.push(chunk); - if cardinality >= self.config.max_batch_rows { - self.flush().await?; + async fn write_batch_non_append_only(&mut self, chunk: StreamChunk) -> Result<()> { + let mut transaction = self.client.transaction().await?; + // 1d flattened array of parameters to be inserted. + let mut insert_parameter_buffer = ParameterBuffer::new( + &self.schema_types, + chunk.cardinality() * chunk.data_types().len(), + ); + let mut delete_parameter_buffer = ParameterBuffer::new( + &self.schema_types, + chunk.cardinality() * self.pk_indices.len(), + ); + // 1d flattened array of parameters to be deleted. + for (op, row) in chunk.rows() { + match op { + Op::UpdateInsert | Op::UpdateDelete => { + bail!("UpdateInsert and UpdateDelete should have been normalized by the sink executor") + } + Op::Insert => { + insert_parameter_buffer.add_row(row); + } + Op::Delete => { + delete_parameter_buffer.add_row(row.project(&self.pk_indices)); + } + } } - Ok(()) - } - async fn barrier(&mut self, is_checkpoint: bool) -> Result { - if is_checkpoint { - self.flush().await?; - } - Ok(()) - } + let (delete_parameters, delete_remaining_parameter) = delete_parameter_buffer.into_parts(); + Self::execute_parameter( + Op::Delete, + &mut transaction, + &self.schema, + &self.config.table, + &self.pk_indices, + delete_parameters, + delete_remaining_parameter, + ) + .await?; + let (insert_parameters, insert_remaining_parameter) = insert_parameter_buffer.into_parts(); + Self::execute_parameter( + Op::Insert, + &mut transaction, + &self.schema, + &self.config.table, + &self.pk_indices, + insert_parameters, + insert_remaining_parameter, + ) + .await?; + transaction.commit().await?; - async fn abort(&mut self) -> Result<()> { Ok(()) } - async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc) -> Result<()> { + async fn execute_parameter( + op: Op, + transaction: &mut tokio_postgres::Transaction<'_>, + schema: &Schema, + table_name: &str, + pk_indices: &[usize], + parameters: Vec>>, + remaining_parameter: Vec>, + ) -> Result<()> { + let column_length = schema.fields().len(); + if !parameters.is_empty() { + let parameter_length = parameters[0].len(); + let rows_length = parameter_length / column_length; + assert_eq!( + parameter_length % column_length, + 0, + "flattened parameters are unaligned" + ); + let statement = match op { + Op::Insert => create_insert_sql(schema, table_name, rows_length), + Op::Delete => create_delete_sql(schema, table_name, pk_indices, rows_length), + _ => unreachable!(), + }; + let statement = transaction.prepare(&statement).await?; + for parameter in parameters { + transaction.execute_raw(&statement, parameter).await?; + } + } + if !remaining_parameter.is_empty() { + let rows_length = remaining_parameter.len() / column_length; + assert_eq!( + remaining_parameter.len() % column_length, + 0, + "flattened parameters are unaligned" + ); + let statement = match op { + Op::Insert => create_insert_sql(schema, table_name, rows_length), + Op::Delete => create_delete_sql(schema, table_name, pk_indices, rows_length), + _ => unreachable!(), + }; + let statement = transaction.prepare(&statement).await?; + transaction + .execute_raw(&statement, remaining_parameter) + .await?; + } Ok(()) } } -fn create_insert_sql(schema: &Schema, table_name: &str) -> String { - let columns: String = schema - .fields() - .iter() - .map(|field| field.name.clone()) - .collect_vec() - .join(", "); - let parameters: String = (0..schema.fields().len()) - .map(|i| format!("${}", i + 1)) - .collect_vec() - .join(", "); - format!("INSERT INTO {table_name} ({columns}) VALUES ({parameters})") +#[async_trait] +impl LogSinker for PostgresSinkWriter { + async fn consume_log_and_sink(mut self, log_reader: &mut impl SinkLogReader) -> Result { + loop { + let (epoch, item) = log_reader.next_item().await?; + match item { + LogStoreReadItem::StreamChunk { chunk, chunk_id } => { + self.write_batch(chunk).await?; + log_reader.truncate(TruncateOffset::Chunk { epoch, chunk_id })?; + } + LogStoreReadItem::Barrier { .. } => { + log_reader.truncate(TruncateOffset::Barrier { epoch })?; + } + LogStoreReadItem::UpdateVnodeBitmap(_) => {} + } + } + } } -fn create_upsert_sql(schema: &Schema, table_name: &str, pk_indices: &[usize]) -> String { +fn create_insert_sql(schema: &Schema, table_name: &str, number_of_rows: usize) -> String { + let number_of_columns = schema.len(); let columns: String = schema .fields() .iter() .map(|field| field.name.clone()) .collect_vec() .join(", "); - let parameters: String = (0..schema.fields().len()) - .map(|i| format!("${}", i + 1)) - .collect_vec() - .join(", "); - let pk_columns = pk_indices - .iter() - .map(|i| schema.fields()[*i].name.clone()) - .collect_vec() - .join(", "); - let update_parameters: String = (0..schema.fields().len()) - .filter(|i| !pk_indices.contains(i)) + let parameters: String = (0..number_of_rows) .map(|i| { - let column = schema.fields()[i].name.clone(); - let param = format!("${}", i + 1); - format!("{column} = {param}") + let row_parameters = (0..number_of_columns) + .map(|j| format!("${}", i * number_of_columns + j + 1)) + .collect_vec() + .join(", "); + format!("({row_parameters})") }) .collect_vec() .join(", "); - format!("INSERT INTO {table_name} ({columns}) VALUES ({parameters}) on conflict ({pk_columns}) do update set {update_parameters}") + format!("INSERT INTO {table_name} ({columns}) VALUES {parameters}") } -fn create_delete_sql(schema: &Schema, table_name: &str, pk_indices: &[usize]) -> String { - let parameters: String = pk_indices - .iter() - .map(|i| format!("{} = ${}", schema.fields()[*i].name, i + 1)) +fn create_delete_sql( + schema: &Schema, + table_name: &str, + pk_indices: &[usize], + number_of_rows: usize, +) -> String { + let number_of_pk = pk_indices.len(); + let parameters: String = (0..number_of_rows) + .map(|i| { + let row_parameters: String = pk_indices + .iter() + .enumerate() + .map(|(j, pk_index)| { + format!( + "{} = ${}", + schema.fields()[*pk_index].name, + i * number_of_pk + j + 1 + ) + }) + .collect_vec() + .join(" AND "); + format!("({row_parameters})") + }) .collect_vec() - .join(" AND "); + .join(" OR "); format!("DELETE FROM {table_name} WHERE {parameters}") } @@ -568,10 +612,10 @@ mod tests { }, ]); let table_name = "test_table"; - let sql = create_insert_sql(&schema, table_name); + let sql = create_insert_sql(&schema, table_name, 3); check( sql, - expect!["INSERT INTO test_table (a, b) VALUES ($1, $2)"], + expect!["INSERT INTO test_table (a, b) VALUES ($1, $2), ($3, $4), ($5, $6)"], ); } @@ -592,31 +636,10 @@ mod tests { }, ]); let table_name = "test_table"; - let sql = create_delete_sql(&schema, table_name, &[1]); - check(sql, expect!["DELETE FROM test_table WHERE b = $2"]); - } - - #[test] - fn test_create_upsert_sql() { - let schema = Schema::new(vec![ - Field { - data_type: DataType::Int32, - name: "a".to_owned(), - sub_fields: vec![], - type_name: "".to_owned(), - }, - Field { - data_type: DataType::Int32, - name: "b".to_owned(), - sub_fields: vec![], - type_name: "".to_owned(), - }, - ]); - let table_name = "test_table"; - let sql = create_upsert_sql(&schema, table_name, &[1]); + let sql = create_delete_sql(&schema, table_name, &[1], 3); check( sql, - expect!["INSERT INTO test_table (a, b) VALUES ($1, $2) on conflict (b) do update set a = $1"], + expect!["DELETE FROM test_table WHERE (b = $1) OR (b = $2) OR (b = $3)"], ); } } diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs index 76e65418cdf18..e9b00a1a63e4f 100644 --- a/src/connector/src/sink/remote.rs +++ b/src/connector/src/sink/remote.rs @@ -782,7 +782,7 @@ impl EmbeddedConnectorClient { match result { Ok(_) => { - tracing::info!("end of jni call {}::{}", class_name, method_name); + tracing::debug!("end of jni call {}::{}", class_name, method_name); } Err(e) => { tracing::error!(error = %e.as_report(), "jni call error"); diff --git a/src/connector/src/source/kafka/enumerator/client.rs b/src/connector/src/source/kafka/enumerator/client.rs index f77cfac1b9bbe..295450b6907e3 100644 --- a/src/connector/src/source/kafka/enumerator/client.rs +++ b/src/connector/src/source/kafka/enumerator/client.rs @@ -20,12 +20,11 @@ use anyhow::{anyhow, Context}; use async_trait::async_trait; use moka::future::Cache as MokaCache; use moka::ops::compute::Op; -use prometheus::core::{AtomicI64, GenericGauge}; use rdkafka::consumer::{BaseConsumer, Consumer}; use rdkafka::error::KafkaResult; use rdkafka::{ClientConfig, Offset, TopicPartitionList}; use risingwave_common::bail; -use risingwave_common::metrics::LabelGuardedMetric; +use risingwave_common::metrics::LabelGuardedIntGauge; use crate::error::{ConnectorError, ConnectorResult}; use crate::source::base::SplitEnumerator; @@ -60,7 +59,7 @@ pub struct KafkaSplitEnumerator { stop_offset: KafkaEnumeratorOffset, sync_call_timeout: Duration, - high_watermark_metrics: HashMap, 2>>, + high_watermark_metrics: HashMap>, } impl KafkaSplitEnumerator {} diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs index 4c8a78adbfec4..a91407e61e767 100644 --- a/src/connector/src/source/kafka/source/reader.rs +++ b/src/connector/src/source/kafka/source/reader.rs @@ -21,12 +21,11 @@ use anyhow::Context; use async_trait::async_trait; use futures::StreamExt; use futures_async_stream::try_stream; -use prometheus::core::{AtomicI64, GenericGauge}; use rdkafka::config::RDKafkaLogLevel; use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::{ClientConfig, Message, Offset, TopicPartitionList}; -use risingwave_common::metrics::LabelGuardedMetric; +use risingwave_common::metrics::LabelGuardedIntGauge; use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; use crate::error::ConnectorResult as Result; @@ -247,10 +246,8 @@ impl KafkaSplitReader { ) }); - let mut latest_message_id_metrics: HashMap< - String, - LabelGuardedMetric, 3>, - > = HashMap::new(); + let mut latest_message_id_metrics: HashMap> = + HashMap::new(); #[for_await] 'for_outer_loop: for msgs in self.consumer.stream().ready_chunks(max_chunk_size) { diff --git a/src/connector/src/source/kafka/stats.rs b/src/connector/src/source/kafka/stats.rs index 7a36c4d1fffea..9e8285a8cc486 100644 --- a/src/connector/src/source/kafka/stats.rs +++ b/src/connector/src/source/kafka/stats.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use prometheus::core::AtomicU64; use prometheus::Registry; use rdkafka::statistics::{Broker, ConsumerGroup, Partition, Topic, Window}; use rdkafka::Statistics; diff --git a/src/connector/with_options/Cargo.toml b/src/connector/with_options/Cargo.toml index 59dd4291e12b7..b8d1ecf510379 100644 --- a/src/connector/with_options/Cargo.toml +++ b/src/connector/with_options/Cargo.toml @@ -9,7 +9,6 @@ repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -proc-macro2 = "1" quote = "1" syn = "2" @@ -18,11 +17,5 @@ proc-macro = true [dev-dependencies] -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [lints] workspace = true diff --git a/src/ctl/Cargo.toml b/src/ctl/Cargo.toml index 7b0fc940de946..cdb8f5377c75e 100644 --- a/src/ctl/Cargo.toml +++ b/src/ctl/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" bytes = "1" @@ -24,7 +18,6 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] } hex = "0.4" inquire = "0.7.0" itertools = { workspace = true } -memcomparable = "0.2" prost = { workspace = true } regex = "1.10.0" risingwave_common = { workspace = true } @@ -33,7 +26,6 @@ risingwave_frontend = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_meta = { workspace = true } risingwave_meta_model = { workspace = true } -risingwave_meta_model_migration = { workspace = true } risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } diff --git a/src/dml/Cargo.toml b/src/dml/Cargo.toml index df0ce4ebc19d6..1ad438db96259 100644 --- a/src/dml/Cargo.toml +++ b/src/dml/Cargo.toml @@ -7,24 +7,13 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] futures = { version = "0.3", default-features = false, features = ["alloc"] } futures-async-stream = { workspace = true } itertools = { workspace = true } parking_lot = { workspace = true } -rand = { workspace = true } risingwave_common = { workspace = true } -risingwave_connector = { workspace = true } -risingwave_pb = { workspace = true } -rw_futures_util = { workspace = true } thiserror = "1" -thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", "rt-multi-thread", @@ -41,7 +30,6 @@ workspace-hack = { path = "../workspace-hack" } [dev-dependencies] assert_matches = "1" -criterion = { workspace = true, features = ["async_tokio"] } paste = "1" tempfile = "3" diff --git a/src/error/Cargo.toml b/src/error/Cargo.toml index 4a99711db6c41..8407197a88769 100644 --- a/src/error/Cargo.toml +++ b/src/error/Cargo.toml @@ -10,7 +10,6 @@ repository = { workspace = true } [dependencies] anyhow = "1" bincode = "1" -bytes = "1" easy-ext = "1" serde = "1" serde-error = "0.1" diff --git a/src/expr/core/Cargo.toml b/src/expr/core/Cargo.toml index 9851186260e5a..16db4ea8f2691 100644 --- a/src/expr/core/Cargo.toml +++ b/src/expr/core/Cargo.toml @@ -9,18 +9,11 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack", "ctor"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack", "ctor"] - [dependencies] anyhow = "1" async-trait = "0.1" auto_impl = "1" await-tree = { workspace = true } -cfg-or-panic = "0.2" chrono = { version = "0.4", default-features = false, features = [ "clock", "std", diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml index ec6fa8afb9e99..30bd6f61d6b95 100644 --- a/src/expr/impl/Cargo.toml +++ b/src/expr/impl/Cargo.toml @@ -10,13 +10,10 @@ repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [package.metadata.cargo-machete] -ignored = ["workspace-hack", "ctor"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack", "ctor"] +ignored = ["chrono-tz", "futures-async-stream"] [features] -external-udf = ["arrow-udf-flight", "arrow-flight", "tonic"] +external-udf = ["arrow-udf-flight", "arrow-flight", "tonic", "ginepro"] js-udf = ["arrow-udf-js"] python-udf = ["arrow-udf-python"] wasm-udf = ["arrow-udf-wasm", "zstd"] @@ -42,7 +39,7 @@ educe = "0.6" fancy-regex = "0.14" futures-async-stream = { workspace = true } futures-util = "0.3" -ginepro = "0.8" +ginepro = { version = "0.8", optional = true } hex = "0.4" hmac = "0.12" iceberg = { workspace = true } diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 3041a4320dd42..74fafdd6d7d97 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" arc-swap = "1" @@ -71,7 +65,6 @@ risingwave_dml = { workspace = true } risingwave_expr = { workspace = true } risingwave_frontend_macro = { path = "macro" } risingwave_hummock_sdk = { workspace = true } -risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } @@ -108,7 +101,6 @@ tower-http = { version = "0.6", features = [ tracing = "0.1" url = "2.5.0" uuid = "1" -zstd = { version = "0.13", default-features = false } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../workspace-hack" } diff --git a/src/frontend/planner_test/Cargo.toml b/src/frontend/planner_test/Cargo.toml index 49ad1876afff0..ee8d4af18655d 100644 --- a/src/frontend/planner_test/Cargo.toml +++ b/src/frontend/planner_test/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" expect-test = "1" diff --git a/src/frontend/planner_test/tests/testdata/input/range_scan.yaml b/src/frontend/planner_test/tests/testdata/input/range_scan.yaml index 52863fa5aaf77..374da681d0987 100644 --- a/src/frontend/planner_test/tests/testdata/input/range_scan.yaml +++ b/src/frontend/planner_test/tests/testdata/input/range_scan.yaml @@ -6,182 +6,184 @@ date INTEGER); CREATE MATERIALIZED VIEW orders_count_by_user AS SELECT user_id, date, count(*) AS orders_count FROM orders GROUP BY user_id, date; + CREATE MATERIALIZED VIEW orders_count_by_user_desc AS + SELECT user_id, date, count(*) AS orders_count FROM orders GROUP BY user_id, date ORDER BY user_id DESC; expected_outputs: [] - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id < 43 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 + 1 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1/0 expected_outputs: - - batch_error + - batch_error - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 2147483647 + 1 expected_outputs: - - batch_error + - batch_error - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 'a' expected_outputs: - - batch_error + - batch_error - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id > 'a' expected_outputs: - - batch_error + - batch_error - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = '42' expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = NULL expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id IS NULL expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id > NULL expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date = 1111 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id > 42 AND date = 1111 AND 2>1 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE date > 1111 AND user_id = 42 AND 5<6 AND date <= 6666 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in (42, 43) expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in (42+1, 44-1) expected_outputs: - - batch_plan + - batch_plan - name: If the IN list has a larger type than the column, the InputRef is casted. Currently this case is not converted to scan range yet. before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in (42.0, 43.0) expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in ('42', '43') expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in ('42', '43.0') expected_outputs: - - batch_error + - batch_error - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in (2147483648, 2147483649) AND date = 6666 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, 3333) expected_outputs: - - batch_plan + - batch_plan - name: test duplicate value in in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, 2222) expected_outputs: - - batch_plan + - batch_plan - name: test NULL in in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, NULL) expected_outputs: - - batch_plan + - batch_plan - name: test NULL in in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (NULL) expected_outputs: - - batch_plan + - batch_plan - name: test multiple in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, 3333) AND date in (4444, 3333) expected_outputs: - - batch_plan + - batch_plan - name: test eq & in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, 3333) AND date = 3333 expected_outputs: - - batch_plan + - batch_plan - name: test eq & in-list before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date in (2222, 3333) AND date = 4444 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id in (2147483648, 2147483649) AND date in (2222, 3333) expected_outputs: - - batch_plan + - batch_plan - id: create_table_and_mv_ordered sql: | CREATE TABLE orders ( @@ -194,93 +196,93 @@ ORDER BY orders_count; expected_outputs: [] - before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE user_id = 42 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE user_id > 42 AND orders_count = 10 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count = 10 expected_outputs: - - batch_plan + - batch_plan - name: merge mutiple upper bound before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count < 10 and orders_count < 30 expected_outputs: - - batch_plan + - batch_plan - name: merge include and exclude upper bound of same value before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count < 10 and orders_count <= 10 expected_outputs: - - batch_plan + - batch_plan - name: merge mutiple lower bound before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count > 10 and orders_count > 30 expected_outputs: - - batch_plan + - batch_plan - name: merge include and exclude lower bound of same value before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count > 10 and orders_count >= 10 expected_outputs: - - batch_plan + - batch_plan - name: invalid range before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count > 10 and orders_count < 5 expected_outputs: - - batch_plan + - batch_plan - name: merge cmp and eq condition before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count > 20 and orders_count < 30 and orders_count = 25 expected_outputs: - - batch_plan + - batch_plan - name: invalid range of merging cmp and eq condition before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count > 20 and orders_count < 30 and orders_count = 35 expected_outputs: - - batch_plan + - batch_plan - name: merge cmp and const-in condition before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count in (10,20,30,40) and orders_count <30 expected_outputs: - - batch_plan + - batch_plan - name: invalid range of merging cmp and const-in condition before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count in (10,20,30,40) and orders_count > 50 expected_outputs: - - batch_plan + - batch_plan - name: merge null and cmp condition before: - - create_table_and_mv_ordered + - create_table_and_mv_ordered sql: | SELECT * FROM orders_count_by_user_ordered WHERE orders_count is null and orders_count < 30 expected_outputs: - - batch_plan + - batch_plan - id: create_small sql: | CREATE TABLE t(x smallint); @@ -289,142 +291,241 @@ expected_outputs: [] - name: When the constant with larger type is out of the range of the column's type, we can convert it as false condition. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x = 60000; expected_outputs: - - batch_plan + - batch_plan - name: When the constant with larger type is out of the upper bound of the column's type, we can convert < as true condition. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x < 60000; expected_outputs: - - batch_plan + - batch_plan - name: When the constant with larger type is out of the upper bound of the column's type, we can convert > as false condition. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x > 60000; expected_outputs: - - batch_plan + - batch_plan - name: When the constant with larger type is out of the lower bound of the column's type, we can convert < as false condition. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x < -60000; expected_outputs: - - batch_plan + - batch_plan - name: When the constant with larger type is out of the lower bound of the column's type, we can convert > as true condition. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x > -60000; expected_outputs: - - batch_plan + - batch_plan - name: When the constant with larger type is in range of the column's type, we can convert it. before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x < 3::bigint and x > 1::bigint; expected_outputs: - - batch_plan + - batch_plan - name: Can't push down the in-compatitble numeric type before: - - create_small + - create_small sql: | SELECT * FROM mv WHERE x = 3.4; expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 or user_id = 2; expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE (user_id = 1) or (user_id = 2 and date = 2222); expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE (user_id = 1) or (user_id = 2 and date in (1111, 2222)); expected_outputs: - - batch_plan + - batch_plan - name: When one arm of or clause contains other conditions, we can't convert it to scan range yet. before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE (user_id = 1) or (user_id = 2 and date in (1111, 2222)) or (user_id != 3); expected_outputs: - - batch_plan + - batch_plan - name: When any arm of or clause is not equal type, we can't convert it to scan range yet. before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id > 1 or user_id < 10 expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 or user_id is null expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 and user_id is null expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 or (user_id is null and date = 1111) expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 or (user_id = 2 and date is null) expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id = 1 or (user_id is null and date is null) expected_outputs: - - batch_plan + - batch_plan - before: - - create_table_and_mv + - create_table_and_mv sql: | SELECT * FROM orders_count_by_user WHERE user_id is null or (user_id is null and date is null) expected_outputs: - - batch_plan + - batch_plan - sql: | create table sbtest1(id INT, k INT, c VARCHAR, pad VARCHAR); create index k1 on sbtest1(k); select count(k) from sbtest1 where k between 0 and 5; expected_outputs: - - batch_plan + - batch_plan - sql: | create table sbtest1(id INT, k INT, c VARCHAR, pad VARCHAR); create index k1 on sbtest1(k); select count(k) from sbtest1 where k between 0 and 500; expected_outputs: - - batch_plan + - batch_plan - sql: | create table sbtest1(id INT, k INT, c VARCHAR, pad VARCHAR, primary key(id)); create index k1 on sbtest1(k); select count(k) from sbtest1 where id between 0 and 5; expected_outputs: - - batch_plan + - batch_plan - sql: | create table t (k int primary key, v int); select v from t where k = 2147483648; -- out of range of int32 expected_outputs: - - logical_plan - - batch_plan + - logical_plan + - batch_plan +- name: When OR clauses contain non-overlapping conditions,, we can pushdown serveral scan_range. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 20); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 20); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id >= 10); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id >= 10); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 20) or (user_id = 15); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 20) or (user_id = 15); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id < 30) or (user_id >= 30 and user_id < 40); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id < 30) or (user_id >= 30 and user_id < 40); + expected_outputs: + - batch_plan +- name: When OR clauses contain overlapping conditions, we can merge serveral scan_range and pushdown. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id <= 30) or (user_id >= 30 and user_id < 40) or (user_id = 15); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id <= 30) or (user_id >= 30 and user_id < 40) or (user_id = 15); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 30) or (user_id > 5 and user_id < 15); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 30) or (user_id > 5 and user_id < 15); + expected_outputs: + - batch_plan +- name: When OR clauses contain overlapping conditions, we cannot push down if it results in a full table scan. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 20) or (user_id > 10); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 20) or (user_id > 10); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 20) or (user_id != 10); + expected_outputs: + - batch_plan +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 20) or (user_id != 10); + expected_outputs: + - batch_plan diff --git a/src/frontend/planner_test/tests/testdata/input/row_filter.yaml b/src/frontend/planner_test/tests/testdata/input/row_filter.yaml index 2b1966f4316fa..7293cadf7512b 100644 --- a/src/frontend/planner_test/tests/testdata/input/row_filter.yaml +++ b/src/frontend/planner_test/tests/testdata/input/row_filter.yaml @@ -22,4 +22,10 @@ create table t(v1 int, v2 int, v3 int, primary key(v1,v2,v3)); select * from t where (v1,v2,v1) > (1,2,3); expected_outputs: + - batch_plan +- sql: | + create table t1(v1 int, v2 int, v3 int); + create materialized view mv1 as select * from t1 order by v1 asc, v2 asc, v3 desc; + select * from mv1 where (v1,v2,v3) > (1,3,1); + expected_outputs: - batch_plan \ No newline at end of file diff --git a/src/frontend/planner_test/tests/testdata/output/range_scan.yaml b/src/frontend/planner_test/tests/testdata/output/range_scan.yaml index ed8b4b863fe64..e175558038c81 100644 --- a/src/frontend/planner_test/tests/testdata/output/range_scan.yaml +++ b/src/frontend/planner_test/tests/testdata/output/range_scan.yaml @@ -7,6 +7,8 @@ date INTEGER); CREATE MATERIALIZED VIEW orders_count_by_user AS SELECT user_id, date, count(*) AS orders_count FROM orders GROUP BY user_id, date; + CREATE MATERIALIZED VIEW orders_count_by_user_desc AS + SELECT user_id, date, count(*) AS orders_count FROM orders GROUP BY user_id, date ORDER BY user_id DESC; - before: - create_table_and_mv sql: | @@ -503,3 +505,134 @@ └─LogicalFilter { predicate: (t.k = 2147483648:Int64) } └─LogicalScan { table: t, columns: [t.k, t.v, t._rw_timestamp] } batch_plan: 'BatchValues { rows: [] }' +- name: When OR clauses contain non-overlapping conditions,, we can pushdown serveral scan_range. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 20); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user.user_id < 10:Int32) OR (orders_count_by_user.user_id > 20:Int32)) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id < Int64(10), orders_count_by_user.user_id > Int64(20)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 20); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user_desc.user_id < 10:Int32) OR (orders_count_by_user_desc.user_id > 20:Int32)) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id < Int64(10), orders_count_by_user_desc.user_id > Int64(20)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id >= 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user.user_id < 10:Int32) OR (orders_count_by_user.user_id >= 10:Int32)) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id < Int64(10), orders_count_by_user.user_id >= Int64(10)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id >= 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user_desc.user_id < 10:Int32) OR (orders_count_by_user_desc.user_id >= 10:Int32)) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id < Int64(10), orders_count_by_user_desc.user_id >= Int64(10)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 20) or (user_id = 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: (((orders_count_by_user.user_id < 10:Int32) OR (orders_count_by_user.user_id > 20:Int32)) OR (orders_count_by_user.user_id = 15:Int32)) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id < Int64(10), orders_count_by_user.user_id = Int64(15), orders_count_by_user.user_id > Int64(20)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 20) or (user_id = 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: (((orders_count_by_user_desc.user_id < 10:Int32) OR (orders_count_by_user_desc.user_id > 20:Int32)) OR (orders_count_by_user_desc.user_id = 15:Int32)) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id < Int64(10), orders_count_by_user_desc.user_id = Int64(15), orders_count_by_user_desc.user_id > Int64(20)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id < 30) or (user_id >= 30 and user_id < 40); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((((orders_count_by_user.user_id < 10:Int32) AND (orders_count_by_user.user_id > 1:Int32)) OR ((orders_count_by_user.user_id > 20:Int32) AND (orders_count_by_user.user_id < 30:Int32))) OR ((orders_count_by_user.user_id >= 30:Int32) AND (orders_count_by_user.user_id < 40:Int32))) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id > Int64(1) AND orders_count_by_user.user_id < Int64(10), orders_count_by_user.user_id > Int64(20) AND orders_count_by_user.user_id < Int64(30), orders_count_by_user.user_id >= Int64(30) AND orders_count_by_user.user_id < Int64(40)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id < 30) or (user_id >= 30 and user_id < 40); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((((orders_count_by_user_desc.user_id < 10:Int32) AND (orders_count_by_user_desc.user_id > 1:Int32)) OR ((orders_count_by_user_desc.user_id > 20:Int32) AND (orders_count_by_user_desc.user_id < 30:Int32))) OR ((orders_count_by_user_desc.user_id >= 30:Int32) AND (orders_count_by_user_desc.user_id < 40:Int32))) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id > Int64(1) AND orders_count_by_user_desc.user_id < Int64(10), orders_count_by_user_desc.user_id > Int64(20) AND orders_count_by_user_desc.user_id < Int64(30), orders_count_by_user_desc.user_id >= Int64(30) AND orders_count_by_user_desc.user_id < Int64(40)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- name: When OR clauses contain overlapping conditions, we can merge serveral scan_range and pushdown. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id <= 30) or (user_id >= 30 and user_id < 40) or (user_id = 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((((orders_count_by_user.user_id < 10:Int32) AND (orders_count_by_user.user_id > 1:Int32)) OR ((orders_count_by_user.user_id > 20:Int32) AND (orders_count_by_user.user_id <= 30:Int32))) OR (((orders_count_by_user.user_id >= 30:Int32) AND (orders_count_by_user.user_id < 40:Int32)) OR (orders_count_by_user.user_id = 15:Int32))) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id > Int64(1) AND orders_count_by_user.user_id < Int64(10), orders_count_by_user.user_id = Int64(15), orders_count_by_user.user_id >= Int64(20) AND orders_count_by_user.user_id <= Int64(40)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10 and user_id > 1) or (user_id > 20 and user_id <= 30) or (user_id >= 30 and user_id < 40) or (user_id = 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((((orders_count_by_user_desc.user_id < 10:Int32) AND (orders_count_by_user_desc.user_id > 1:Int32)) OR ((orders_count_by_user_desc.user_id > 20:Int32) AND (orders_count_by_user_desc.user_id <= 30:Int32))) OR (((orders_count_by_user_desc.user_id >= 30:Int32) AND (orders_count_by_user_desc.user_id < 40:Int32)) OR (orders_count_by_user_desc.user_id = 15:Int32))) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id > Int64(1) AND orders_count_by_user_desc.user_id < Int64(10), orders_count_by_user_desc.user_id = Int64(15), orders_count_by_user_desc.user_id >= Int64(20) AND orders_count_by_user_desc.user_id <= Int64(40)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 10) or (user_id > 30) or (user_id > 5 and user_id < 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: (((orders_count_by_user.user_id < 10:Int32) OR (orders_count_by_user.user_id > 30:Int32)) OR ((orders_count_by_user.user_id > 5:Int32) AND (orders_count_by_user.user_id < 15:Int32))) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], scan_ranges: [orders_count_by_user.user_id <= Int64(15), orders_count_by_user.user_id > Int64(30)], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 10) or (user_id > 30) or (user_id > 5 and user_id < 15); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: (((orders_count_by_user_desc.user_id < 10:Int32) OR (orders_count_by_user_desc.user_id > 30:Int32)) OR ((orders_count_by_user_desc.user_id > 5:Int32) AND (orders_count_by_user_desc.user_id < 15:Int32))) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], scan_ranges: [orders_count_by_user_desc.user_id <= Int64(15), orders_count_by_user_desc.user_id > Int64(30)], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- name: When OR clauses contain overlapping conditions, we cannot push down if it results in a full table scan. + before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 20) or (user_id > 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user.user_id < 20:Int32) OR (orders_count_by_user.user_id > 10:Int32)) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 20) or (user_id > 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user_desc.user_id < 20:Int32) OR (orders_count_by_user_desc.user_id > 10:Int32)) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user WHERE (user_id < 20) or (user_id != 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user.user_id < 20:Int32) OR (orders_count_by_user.user_id <> 10:Int32)) } + └─BatchScan { table: orders_count_by_user, columns: [orders_count_by_user.user_id, orders_count_by_user.date, orders_count_by_user.orders_count], distribution: UpstreamHashShard(orders_count_by_user.user_id, orders_count_by_user.date) } +- before: + - create_table_and_mv + sql: | + SELECT * FROM orders_count_by_user_desc WHERE (user_id < 20) or (user_id != 10); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: ((orders_count_by_user_desc.user_id < 20:Int32) OR (orders_count_by_user_desc.user_id <> 10:Int32)) } + └─BatchScan { table: orders_count_by_user_desc, columns: [orders_count_by_user_desc.user_id, orders_count_by_user_desc.date, orders_count_by_user_desc.orders_count], distribution: UpstreamHashShard(orders_count_by_user_desc.user_id, orders_count_by_user_desc.date) } diff --git a/src/frontend/planner_test/tests/testdata/output/row_filter.yaml b/src/frontend/planner_test/tests/testdata/output/row_filter.yaml index 1ef80b8e025bd..5de2e79815476 100644 --- a/src/frontend/planner_test/tests/testdata/output/row_filter.yaml +++ b/src/frontend/planner_test/tests/testdata/output/row_filter.yaml @@ -17,7 +17,7 @@ batch_plan: |- BatchExchange { order: [], dist: Single } └─BatchFilter { predicate: (Row(t.v1, t.v3) > '(2,3)':Struct(StructType { field_names: [], field_types: [Int32, Int32] })) } - └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], scan_ranges: [t.v1 > Int32(2)], distribution: UpstreamHashShard(t.v1, t.v2, t.v3) } + └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], scan_ranges: [t.v1 >= Int32(2)], distribution: UpstreamHashShard(t.v1, t.v2, t.v3) } - sql: | create table t(v1 int, v2 int, v3 int, primary key(v1,v2,v3)); select * from t where (v3,v2,v1) > (1,2,3); @@ -31,4 +31,12 @@ batch_plan: |- BatchExchange { order: [], dist: Single } └─BatchFilter { predicate: (Row(t.v1, t.v2, t.v1) > '(1,2,3)':Struct(StructType { field_names: [], field_types: [Int32, Int32, Int32] })) } - └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], scan_ranges: [(t.v1, t.v2) > (Int32(1), Int32(2))], distribution: UpstreamHashShard(t.v1, t.v2, t.v3) } + └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], scan_ranges: [(t.v1, t.v2) >= (Int32(1), Int32(2))], distribution: UpstreamHashShard(t.v1, t.v2, t.v3) } +- sql: | + create table t1(v1 int, v2 int, v3 int); + create materialized view mv1 as select * from t1 order by v1 asc, v2 asc, v3 desc; + select * from mv1 where (v1,v2,v3) > (1,3,1); + batch_plan: |- + BatchExchange { order: [], dist: Single } + └─BatchFilter { predicate: (Row(mv1.v1, mv1.v2, mv1.v3) > '(1,3,1)':Struct(StructType { field_names: [], field_types: [Int32, Int32, Int32] })) } + └─BatchScan { table: mv1, columns: [mv1.v1, mv1.v2, mv1.v3], scan_ranges: [(mv1.v1, mv1.v2) >= (Int32(1), Int32(3))], distribution: SomeShard } diff --git a/src/frontend/src/catalog/system_catalog/information_schema/key_column_usage.rs b/src/frontend/src/catalog/system_catalog/information_schema/key_column_usage.rs new file mode 100644 index 0000000000000..c9343b4edcba1 --- /dev/null +++ b/src/frontend/src/catalog/system_catalog/information_schema/key_column_usage.rs @@ -0,0 +1,59 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::types::Fields; +use risingwave_frontend_macro::system_catalog; + +/// The view `key_column_usage` contains all constraints belonging to tables that the current user owns or has some privilege other than SELECT on. +/// Ref: [`https://www.postgresql.org/docs/current/infoschema-key-column-usage.html`] +/// Limitation: +/// This view assume the constraint schema is the same as the table schema, since `pg_catalog`.`pg_constraint` only support primary key. +#[system_catalog( + view, + "information_schema.key_column_usage", + "WITH key_column_usage_without_name AS ( + SELECT CURRENT_DATABASE() AS constraint_catalog, + pg_namespace.nspname AS constraint_schema, + pg_constraint.conname AS constraint_name, + CURRENT_DATABASE() AS table_catalog, + pg_namespace.nspname AS table_schema, + pg_class.relname AS table_name, + unnest(conkey) as col_id, + conrelid as table_id + FROM pg_catalog.pg_constraint + JOIN pg_catalog.pg_class ON pg_constraint.conrelid = pg_class.oid + JOIN rw_catalog.rw_relations ON rw_relations.id = pg_class.oid + JOIN pg_catalog.pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE rw_relations.relation_type != 'table' or (rw_relations.relation_type = 'table' and has_table_privilege(pg_constraint.conrelid, 'INSERT, UPDATE, DELETE')) + ORDER BY constraint_catalog, constraint_schema, constraint_name + ) + SELECT constraint_catalog, constraint_schema, constraint_name, table_catalog, table_schema, table_name, + name as column_name, rw_columns.position as ordinal_position, NULL::int as position_in_unique_constraint + FROM key_column_usage_without_name + JOIN rw_catalog.rw_columns ON + rw_columns.position = key_column_usage_without_name.col_id AND + rw_columns.relation_id = key_column_usage_without_name.table_id" +)] +#[derive(Fields)] +struct KeyColumnUsage { + constraint_catalog: String, + constraint_schema: String, + constraint_name: String, + table_catalog: String, + table_schema: String, + table_name: String, + column_name: String, + ordinal_position: i32, + position_in_unique_constraint: i32, +} diff --git a/src/frontend/src/catalog/system_catalog/information_schema/mod.rs b/src/frontend/src/catalog/system_catalog/information_schema/mod.rs index 6a81d0c109ec0..c3ca755cce489 100644 --- a/src/frontend/src/catalog/system_catalog/information_schema/mod.rs +++ b/src/frontend/src/catalog/system_catalog/information_schema/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod columns; +mod key_column_usage; mod schemata; mod table_constraints; mod tables; diff --git a/src/frontend/src/catalog/system_catalog/information_schema/table_constraints.rs b/src/frontend/src/catalog/system_catalog/information_schema/table_constraints.rs index 24c3970a02a82..3f9a5059a51bc 100644 --- a/src/frontend/src/catalog/system_catalog/information_schema/table_constraints.rs +++ b/src/frontend/src/catalog/system_catalog/information_schema/table_constraints.rs @@ -18,7 +18,7 @@ use risingwave_frontend_macro::system_catalog; /// The view `table_constraints` contains all constraints belonging to tables that the current user owns or has some privilege other than SELECT on. /// Ref: [`https://www.postgresql.org/docs/current/infoschema-table-constraints.html`] /// Limitation: -/// This view assume the constraint schema is the same as the table schema, since `pg_clatalog`.`pg_constraint` only support primrary key. +/// This view assume the constraint schema is the same as the table schema, since `pg_catalog`.`pg_constraint` only support primary key. #[system_catalog( view, "information_schema.table_constraints", diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs index 25fb0527cddf4..a38c8b2db441a 100644 --- a/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs +++ b/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs @@ -46,6 +46,7 @@ mod pg_settings; mod pg_shadow; mod pg_shdescription; mod pg_stat_activity; +mod pg_stat_user_tables; mod pg_tables; mod pg_tablespace; mod pg_trigger; diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_stat_user_tables.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_stat_user_tables.rs new file mode 100644 index 0000000000000..3aafd29c18312 --- /dev/null +++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_stat_user_tables.rs @@ -0,0 +1,87 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::types::{Fields, Timestamptz}; +use risingwave_frontend_macro::system_catalog; + +/// The `pg_stat_user_tables` view will contain one row for each user table in the current database, +/// showing statistics about accesses to that specific table. +/// Ref: [`https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-ALL-TABLES-VIEW`] +#[system_catalog( + view, + "pg_catalog.pg_stat_user_tables", + "SELECT + rr.id as relid, + rs.name as schemaname, + rr.name as relname, + NULL::bigint as seq_scan, + NULL::timestamptz as last_seq_scan, + NULL::bigint as seq_tup_read, + NULL::bigint as idx_scan, + NULL::timestamptz as last_idx_scan, + NULL::bigint as idx_tup_fetch, + NULL::bigint as n_tup_ins, + NULL::bigint as n_tup_del, + NULL::bigint as n_tup_hot_upd, + NULL::bigint as n_tup_newpage_upd, + rts.total_key_count as n_live_tup, + NULL::bigint as n_dead_tup, + NULL::bigint as n_mod_since_analyze, + NULL::bigint as n_ins_since_vacuum, + NULL::timestamptz as last_vacuum, + NULL::timestamptz as last_autovacuum, + NULL::timestamptz as last_analyze, + NULL::timestamptz as last_autoanalyze, + NULL::bigint as vacuum_count, + NULL::bigint as autovacuum_count, + NULL::bigint as analyze_count, + NULL::bigint as autoanalyze_count + FROM + rw_relations rr + left join rw_table_stats rts on rr.id = rts.id + join rw_schemas rs on schema_id = rs.id + WHERE + rs.name != 'rw_catalog' + AND rs.name != 'pg_catalog' + AND rs.name != 'information_schema' +" +)] +#[derive(Fields)] +struct PgStatUserTables { + relid: i32, + schemaname: String, + relname: String, + seq_scan: i64, + last_seq_scan: Timestamptz, + seq_tup_read: i64, + idx_scan: i64, + last_idx_scan: Timestamptz, + idx_tup_fetch: i64, + n_tup_ins: i64, + n_tup_del: i64, + n_tup_hot_upd: i64, + n_tup_newpage_upd: i64, + n_live_tup: i64, + n_dead_tup: i64, + n_mod_since_analyze: i64, + n_ins_since_vacuum: i64, + last_vacuum: Timestamptz, + last_autovacuum: Timestamptz, + last_analyze: Timestamptz, + last_autoanalyze: Timestamptz, + vacuum_count: i64, + autovacuum_count: i64, + analyze_count: i64, + autoanalyze_count: i64, +} diff --git a/src/frontend/src/expr/user_defined_function.rs b/src/frontend/src/expr/user_defined_function.rs index 084fe7387d766..38d702e94b5b6 100644 --- a/src/frontend/src/expr/user_defined_function.rs +++ b/src/frontend/src/expr/user_defined_function.rs @@ -81,7 +81,7 @@ impl Expr for UserDefinedFunction { ExprNode { function_type: Type::Unspecified.into(), return_type: Some(self.return_type().to_protobuf()), - rex_node: Some(RexNode::Udf(UserDefinedFunction { + rex_node: Some(RexNode::Udf(Box::new(UserDefinedFunction { children: self.args.iter().map(Expr::to_expr_proto).collect(), name: self.catalog.name.clone(), arg_names: self.catalog.arg_names.clone(), @@ -98,7 +98,7 @@ impl Expr for UserDefinedFunction { body: self.catalog.body.clone(), compressed_binary: self.catalog.compressed_binary.clone(), always_retry_on_network_error: self.catalog.always_retry_on_network_error, - })), + }))), } } } diff --git a/src/frontend/src/handler/alter_table_column.rs b/src/frontend/src/handler/alter_table_column.rs index 0342004f6b1b5..04cc9c8f8defa 100644 --- a/src/frontend/src/handler/alter_table_column.rs +++ b/src/frontend/src/handler/alter_table_column.rs @@ -296,10 +296,10 @@ pub(crate) fn hijack_merger_for_target_table( } } - let pb_project = PbNodeBody::Project(ProjectNode { + let pb_project = PbNodeBody::Project(Box::new(ProjectNode { select_list: exprs.iter().map(|expr| expr.to_expr_proto()).collect(), ..Default::default() - }); + })); for fragment in graph.fragments.values_mut() { if let Some(node) = &mut fragment.node { diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs index c89050b331d3f..3502455021ca7 100644 --- a/src/frontend/src/handler/create_sink.rs +++ b/src/frontend/src/handler/create_sink.rs @@ -649,9 +649,9 @@ pub(crate) fn insert_merger_to_union_with_project( // TODO: MergeNode is used as a placeholder, see issue #17658 node.input.push(StreamNode { input: vec![StreamNode { - node_body: Some(NodeBody::Merge(MergeNode { + node_body: Some(NodeBody::Merge(Box::new(MergeNode { ..Default::default() - })), + }))), ..Default::default() }], identity: uniq_identity diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs index 29af1e8c62749..1ea32fa941e6d 100644 --- a/src/frontend/src/handler/create_table.rs +++ b/src/frontend/src/handler/create_table.rs @@ -502,13 +502,10 @@ pub(crate) async fn gen_create_table_plan_with_source( format_encode: FormatEncodeOptions, source_watermarks: Vec, mut col_id_gen: ColumnIdGenerator, - append_only: bool, - on_conflict: Option, - with_version_column: Option, include_column_options: IncludeOption, - engine: Engine, + props: CreateTableProps, ) -> Result<(PlanRef, Option, PbTable)> { - if append_only + if props.append_only && format_encode.format != Format::Plain && format_encode.format != Format::Native { @@ -522,6 +519,9 @@ pub(crate) async fn gen_create_table_plan_with_source( let session = &handler_args.session; let with_properties = bind_connector_props(&handler_args, &format_encode, false)?; + let db_name: &str = session.database(); + let (schema_name, _) = Binder::resolve_schema_qualified_name(db_name, table_name.clone())?; + let (columns_from_resolve_source, source_info) = bind_columns_from_source( session, &format_encode, @@ -556,14 +556,10 @@ pub(crate) async fn gen_create_table_plan_with_source( let (plan, table) = gen_table_plan_with_source( context.into(), + schema_name, source_catalog, - append_only, - on_conflict, - with_version_column, - Some(col_id_gen.into_version()), - database_id, - schema_id, - engine, + col_id_gen.into_version(), + props, )?; Ok((plan, Some(pb_source), table)) @@ -579,13 +575,8 @@ pub(crate) fn gen_create_table_plan( constraints: Vec, mut col_id_gen: ColumnIdGenerator, source_watermarks: Vec, - append_only: bool, - on_conflict: Option, - with_version_column: Option, - webhook_info: Option, - engine: Engine, + props: CreateTableProps, ) -> Result<(PlanRef, PbTable)> { - let definition = context.normalized_sql().to_owned(); let mut columns = bind_sql_columns(&column_defs)?; for c in &mut columns { c.column_desc.column_id = col_id_gen.generate(&*c)?; @@ -602,14 +593,9 @@ pub(crate) fn gen_create_table_plan( columns, column_defs, constraints, - definition, source_watermarks, - append_only, - on_conflict, - with_version_column, - Some(col_id_gen.into_version()), - webhook_info, - engine, + col_id_gen.into_version(), + props, ) } @@ -620,15 +606,11 @@ pub(crate) fn gen_create_table_plan_without_source( columns: Vec, column_defs: Vec, constraints: Vec, - definition: String, source_watermarks: Vec, - append_only: bool, - on_conflict: Option, - with_version_column: Option, - version: Option, - webhook_info: Option, - engine: Engine, + version: TableVersion, + props: CreateTableProps, ) -> Result<(PlanRef, PbTable)> { + // XXX: Why not bind outside? let pk_names = bind_sql_pk_names(&column_defs, bind_table_constraints(&constraints)?)?; let (mut columns, pk_column_ids, row_id_index) = bind_pk_and_row_id_on_relation(columns, pk_names, true)?; @@ -650,89 +632,97 @@ pub(crate) fn gen_create_table_plan_without_source( let session = context.session_ctx().clone(); let db_name = session.database(); - let (schema_name, name) = Binder::resolve_schema_qualified_name(db_name, table_name)?; - let (database_id, schema_id) = - session.get_database_and_schema_id_for_create(schema_name.clone())?; + let (schema_name, table_name) = Binder::resolve_schema_qualified_name(db_name, table_name)?; - gen_table_plan_inner( - context.into(), - name, + let info = CreateTableInfo { columns, pk_column_ids, row_id_index, - definition, watermark_descs, - append_only, - on_conflict, - with_version_column, + source_catalog: None, version, - None, - database_id, - schema_id, - webhook_info, - engine, - ) + }; + + gen_table_plan_inner(context.into(), schema_name, table_name, info, props) } fn gen_table_plan_with_source( context: OptimizerContextRef, + schema_name: Option, source_catalog: SourceCatalog, - append_only: bool, - on_conflict: Option, - with_version_column: Option, - version: Option, /* TODO: this should always be `Some` if we support `ALTER - * TABLE` for `CREATE TABLE AS`. */ - database_id: DatabaseId, - schema_id: SchemaId, - engine: Engine, + version: TableVersion, + props: CreateTableProps, ) -> Result<(PlanRef, PbTable)> { - let cloned_source_catalog = source_catalog.clone(); - gen_table_plan_inner( - context, - source_catalog.name, - source_catalog.columns, - source_catalog.pk_col_ids, - source_catalog.row_id_index, - source_catalog.definition, - source_catalog.watermark_descs, - append_only, - on_conflict, - with_version_column, + let table_name = source_catalog.name.clone(); + + let info = CreateTableInfo { + columns: source_catalog.columns.clone(), + pk_column_ids: source_catalog.pk_col_ids.clone(), + row_id_index: source_catalog.row_id_index, + watermark_descs: source_catalog.watermark_descs.clone(), + source_catalog: Some(source_catalog), version, - Some(cloned_source_catalog), - database_id, - schema_id, - None, - engine, - ) + }; + + gen_table_plan_inner(context, schema_name, table_name, info, props) +} + +/// Arguments of the functions that generate a table plan, part 1. +/// +/// Compared to [`CreateTableProps`], this struct contains fields that need some work of binding +/// or resolving based on the user input. +pub struct CreateTableInfo { + pub columns: Vec, + pub pk_column_ids: Vec, + pub row_id_index: Option, + pub watermark_descs: Vec, + pub source_catalog: Option, + pub version: TableVersion, +} + +/// Arguments of the functions that generate a table plan, part 2. +/// +/// Compared to [`CreateTableInfo`], this struct contains fields that can be (relatively) simply +/// obtained from the input or the context. +pub struct CreateTableProps { + pub definition: String, + pub append_only: bool, + pub on_conflict: Option, + pub with_version_column: Option, + pub webhook_info: Option, + pub engine: Engine, } #[allow(clippy::too_many_arguments)] fn gen_table_plan_inner( context: OptimizerContextRef, + schema_name: Option, table_name: String, - columns: Vec, - pk_column_ids: Vec, - row_id_index: Option, - definition: String, - watermark_descs: Vec, - append_only: bool, - on_conflict: Option, - with_version_column: Option, - version: Option, /* TODO: this should always be `Some` if we support `ALTER - * TABLE` for `CREATE TABLE AS`. */ - source_catalog: Option, - database_id: DatabaseId, - schema_id: SchemaId, - webhook_info: Option, - engine: Engine, + info: CreateTableInfo, + props: CreateTableProps, ) -> Result<(PlanRef, PbTable)> { + let CreateTableInfo { + ref columns, + row_id_index, + ref watermark_descs, + ref source_catalog, + .. + } = info; + let CreateTableProps { + append_only, + on_conflict, + .. + } = props; + + let (database_id, schema_id) = context + .session_ctx() + .get_database_and_schema_id_for_create(schema_name)?; + let session = context.session_ctx().clone(); let retention_seconds = context.with_options().retention_seconds(); - let is_external_source = source_catalog.is_some(); let source_node: PlanRef = LogicalSource::new( - source_catalog.map(|source| Rc::new(source.clone())), + source_catalog.clone().map(Rc::new), columns.clone(), row_id_index, SourceNodeKind::CreateTable, @@ -784,20 +774,11 @@ fn gen_table_plan_inner( let materialize = plan_root.gen_table_plan( context, table_name, - columns, - definition, - pk_column_ids, - row_id_index, - append_only, - on_conflict, - with_version_column, - watermark_descs, - version, - is_external_source, - retention_seconds, - None, - webhook_info, - engine, + info, + CreateTableProps { + on_conflict, + ..props + }, )?; let mut table = materialize.table().to_prost(schema_id, database_id); @@ -919,24 +900,27 @@ pub(crate) fn gen_create_table_plan_for_cdc_table( let materialize = plan_root.gen_table_plan( context, resolved_table_name, - columns, - definition, - pk_column_ids, - None, - false, - on_conflict, - with_version_column, - vec![], - Some(col_id_gen.into_version()), - true, - None, - Some(cdc_table_id), - None, - engine, + CreateTableInfo { + columns, + pk_column_ids, + row_id_index: None, + watermark_descs: vec![], + source_catalog: Some((*source).clone()), + version: col_id_gen.into_version(), + }, + CreateTableProps { + definition, + append_only: false, + on_conflict, + with_version_column, + webhook_info: None, + engine, + }, )?; let mut table = materialize.table().to_prost(schema_id, database_id); table.owner = session.user_id(); + table.cdc_table_id = Some(cdc_table_id); table.dependent_relations = vec![source.id]; Ok((materialize.into(), table)) @@ -1024,6 +1008,18 @@ pub(super) async fn handle_create_table_plan( &include_column_options, &cdc_table_info, )?; + let webhook_info = webhook_info + .map(|info| bind_webhook_info(&handler_args.session, &column_defs, info)) + .transpose()?; + + let props = CreateTableProps { + definition: handler_args.normalized_sql.clone(), + append_only, + on_conflict, + with_version_column: with_version_column.clone(), + webhook_info, + engine, + }; let ((plan, source, table), job_type) = match (format_encode, cdc_table_info.as_ref()) { (Some(format_encode), None) => ( @@ -1037,20 +1033,13 @@ pub(super) async fn handle_create_table_plan( format_encode, source_watermarks, col_id_gen, - append_only, - on_conflict, - with_version_column, include_column_options, - engine, + props, ) .await?, TableJobType::General, ), (None, None) => { - let webhook_info = webhook_info - .map(|info| bind_webhook_info(&handler_args.session, &column_defs, info)) - .transpose()?; - let context = OptimizerContext::new(handler_args, explain_options); let (plan, table) = gen_create_table_plan( context, @@ -1059,11 +1048,7 @@ pub(super) async fn handle_create_table_plan( constraints, col_id_gen, source_watermarks, - append_only, - on_conflict, - with_version_column, - webhook_info, - engine, + props, )?; ((plan, None, table), TableJobType::General) @@ -1784,6 +1769,15 @@ pub async fn generate_stream_graph_for_replace_table( ) -> Result<(StreamFragmentGraph, Table, Option, TableJobType)> { use risingwave_pb::catalog::table::OptionalAssociatedSourceId; + let props = CreateTableProps { + definition: handler_args.normalized_sql.clone(), + append_only, + on_conflict, + with_version_column: with_version_column.clone(), + webhook_info: original_catalog.webhook_info.clone(), + engine, + }; + let ((plan, mut source, table), job_type) = match (format_encode, cdc_table_info.as_ref()) { (Some(format_encode), None) => ( gen_create_table_plan_with_source( @@ -1796,11 +1790,8 @@ pub async fn generate_stream_graph_for_replace_table( format_encode, source_watermarks, col_id_gen, - append_only, - on_conflict, - with_version_column, include_column_options, - engine, + props, ) .await?, TableJobType::General, @@ -1814,11 +1805,7 @@ pub async fn generate_stream_graph_for_replace_table( constraints, col_id_gen, source_watermarks, - append_only, - on_conflict, - with_version_column, - original_catalog.webhook_info.clone(), - engine, + props, )?; ((plan, None, table), TableJobType::General) } diff --git a/src/frontend/src/handler/create_table_as.rs b/src/frontend/src/handler/create_table_as.rs index d90ad6afe9a73..d9019656e4e98 100644 --- a/src/frontend/src/handler/create_table_as.rs +++ b/src/frontend/src/handler/create_table_as.rs @@ -21,7 +21,9 @@ use risingwave_sqlparser::ast::{ColumnDef, ObjectName, OnConflict, Query, Statem use super::{HandlerArgs, RwPgResponse}; use crate::binder::BoundStatement; use crate::error::{ErrorCode, Result}; -use crate::handler::create_table::{gen_create_table_plan_without_source, ColumnIdGenerator}; +use crate::handler::create_table::{ + gen_create_table_plan_without_source, ColumnIdGenerator, CreateTableProps, +}; use crate::handler::query::handle_query; use crate::{build_graph, Binder, OptimizerContext}; pub async fn handle_create_as( @@ -107,14 +109,16 @@ pub async fn handle_create_as( columns, vec![], vec![], - "".to_owned(), // TODO: support `SHOW CREATE TABLE` for `CREATE TABLE AS` - vec![], // No watermark should be defined in for `CREATE TABLE AS` - append_only, - on_conflict, - with_version_column, - Some(col_id_gen.into_version()), - None, - engine, + vec![], // No watermark should be defined in for `CREATE TABLE AS` + col_id_gen.into_version(), + CreateTableProps { + definition: "".to_owned(), // TODO: empty definition means no schema change support + append_only, + on_conflict, + with_version_column, + webhook_info: None, + engine, + }, )?; let graph = build_graph(plan)?; diff --git a/src/frontend/src/monitor/stats.rs b/src/frontend/src/monitor/stats.rs index e958a4e0588e6..c3e716a8f96f8 100644 --- a/src/frontend/src/monitor/stats.rs +++ b/src/frontend/src/monitor/stats.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::sync::{Arc, LazyLock}; use std::time::Duration; -use prometheus::core::{AtomicI64, AtomicU64, GenericCounter, GenericGauge}; +use prometheus::core::{AtomicU64, GenericCounter}; use prometheus::{ exponential_buckets, histogram_opts, register_histogram_vec_with_registry, register_histogram_with_registry, register_int_counter_with_registry, @@ -96,8 +96,8 @@ pub struct CursorMetrics { pub subscription_cursor_query_duration: HistogramVec, pub subscription_cursor_declare_duration: HistogramVec, pub subscription_cursor_fetch_duration: HistogramVec, - subsription_cursor_nums: GenericGauge, - invalid_subsription_cursor_nums: GenericGauge, + subsription_cursor_nums: IntGauge, + invalid_subsription_cursor_nums: IntGauge, subscription_cursor_last_fetch_duration: HistogramVec, _cursor_metrics_collector: Option>, } @@ -199,8 +199,8 @@ struct CursorMetricsCollector { impl CursorMetricsCollector { fn new( session_map: SessionMapRef, - subsription_cursor_nums: GenericGauge, - invalid_subsription_cursor_nums: GenericGauge, + subsription_cursor_nums: IntGauge, + invalid_subsription_cursor_nums: IntGauge, subscription_cursor_last_fetch_duration: HistogramVec, ) -> Self { const COLLECT_INTERVAL_SECONDS: u64 = 60; diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs index 14e45f02517f4..230224557b63b 100644 --- a/src/frontend/src/optimizer/mod.rs +++ b/src/frontend/src/optimizer/mod.rs @@ -51,14 +51,11 @@ pub use optimizer_context::*; use plan_expr_rewriter::ConstEvalRewriter; use property::Order; use risingwave_common::bail; -use risingwave_common::catalog::{ - ColumnCatalog, ColumnDesc, ColumnId, ConflictBehavior, Engine, Field, Schema, -}; +use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, ConflictBehavior, Field, Schema}; use risingwave_common::types::DataType; use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::iter_util::ZipEqDebug; use risingwave_connector::sink::catalog::SinkFormatDesc; -use risingwave_pb::catalog::{PbWebhookSourceInfo, WatermarkDesc}; use risingwave_pb::stream_plan::StreamScanType; use self::heuristic_optimizer::ApplyOrder; @@ -73,9 +70,10 @@ use self::plan_visitor::InputRefValidator; use self::plan_visitor::{has_batch_exchange, CardinalityVisitor, StreamKeyChecker}; use self::property::{Cardinality, RequiredDist}; use self::rule::*; -use crate::catalog::table_catalog::{TableType, TableVersion}; +use crate::catalog::table_catalog::TableType; use crate::error::{ErrorCode, Result}; use crate::expr::TimestamptzExprFinder; +use crate::handler::create_table::{CreateTableInfo, CreateTableProps}; use crate::optimizer::plan_node::generic::{SourceNodeKind, Union}; use crate::optimizer::plan_node::{ BatchExchange, PlanNodeType, PlanTreeNode, RewriteExprsRecursive, StreamExchange, StreamUnion, @@ -639,25 +637,26 @@ impl PlanRoot { } /// Optimize and generate a create table plan. - #[allow(clippy::too_many_arguments)] pub fn gen_table_plan( mut self, context: OptimizerContextRef, table_name: String, - columns: Vec, - definition: String, - pk_column_ids: Vec, - row_id_index: Option, - append_only: bool, - on_conflict: Option, - with_version_column: Option, - watermark_descs: Vec, - version: Option, - with_external_source: bool, - retention_seconds: Option, - cdc_table_id: Option, - webhook_info: Option, - engine: Engine, + CreateTableInfo { + columns, + pk_column_ids, + row_id_index, + watermark_descs, + source_catalog, + version, + }: CreateTableInfo, + CreateTableProps { + definition, + append_only, + on_conflict, + with_version_column, + webhook_info, + engine, + }: CreateTableProps, ) -> Result { assert_eq!(self.phase, PlanPhase::Logical); assert_eq!(self.plan.convention(), Convention::Logical); @@ -751,6 +750,7 @@ impl PlanRoot { None }; + let with_external_source = source_catalog.is_some(); let union_inputs = if with_external_source { let mut external_source_node = stream_plan; external_source_node = @@ -868,6 +868,8 @@ impl PlanRoot { ))? } + let retention_seconds = context.with_options().retention_seconds(); + let table_required_dist = { let mut bitset = FixedBitSet::with_capacity(columns.len()); for idx in &pk_column_indices { @@ -891,7 +893,6 @@ impl PlanRoot { row_id_index, version, retention_seconds, - cdc_table_id, webhook_info, engine, ) diff --git a/src/frontend/src/optimizer/plan_node/stream_asof_join.rs b/src/frontend/src/optimizer/plan_node/stream_asof_join.rs index e4ce2c7edfff2..80d2afd017f81 100644 --- a/src/frontend/src/optimizer/plan_node/stream_asof_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_asof_join.rs @@ -310,7 +310,7 @@ impl StreamNode for StreamAsOfJoin { _ => unreachable!(), }; - NodeBody::AsOfJoin(AsOfJoinNode { + NodeBody::AsOfJoin(Box::new(AsOfJoinNode { join_type: asof_join_type.into(), left_key: left_jk_indices_prost, right_key: right_jk_indices_prost, @@ -321,7 +321,7 @@ impl StreamNode for StreamAsOfJoin { right_deduped_input_pk_indices, output_indices: self.core.output_indices.iter().map(|&x| x as u32).collect(), asof_desc: Some(self.inequality_desc), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs index 7ab2912bd1118..e2d57f9f5626d 100644 --- a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs +++ b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs @@ -190,10 +190,10 @@ impl StreamCdcTableScan { append_only: true, identity: "StreamCdcFilter".to_owned(), fields: cdc_source_schema.clone(), - node_body: Some(PbNodeBody::CdcFilter(CdcFilterNode { + node_body: Some(PbNodeBody::CdcFilter(Box::new(CdcFilterNode { search_condition: Some(filter_expr.to_expr_proto()), upstream_source_id, - })), + }))), }; let exchange_operator_id = self.core.ctx.next_plan_node_id(); @@ -205,13 +205,13 @@ impl StreamCdcTableScan { append_only: true, identity: "Exchange".to_owned(), fields: cdc_source_schema.clone(), - node_body: Some(PbNodeBody::Exchange(ExchangeNode { + node_body: Some(PbNodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(DispatchStrategy { r#type: DispatcherType::Simple as _, dist_key_indices: vec![], // simple exchange doesn't need dist key output_indices: (0..cdc_source_schema.len() as u32).collect(), }), - })), + }))), }; // The required columns from the external table @@ -242,7 +242,7 @@ impl StreamCdcTableScan { ); let options = self.core.options.to_proto(); - let stream_scan_body = PbNodeBody::StreamCdcScan(StreamCdcScanNode { + let stream_scan_body = PbNodeBody::StreamCdcScan(Box::new(StreamCdcScanNode { table_id: upstream_source_id, upstream_column_ids, output_indices, @@ -252,7 +252,7 @@ impl StreamCdcTableScan { rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit, disable_backfill: options.disable_backfill, options: Some(options), - }); + })); // plan: merge -> filter -> exchange(simple) -> stream_scan Ok(PbStreamNode { diff --git a/src/frontend/src/optimizer/plan_node/stream_changelog.rs b/src/frontend/src/optimizer/plan_node/stream_changelog.rs index 34bfdec281815..bf09b060c4f87 100644 --- a/src/frontend/src/optimizer/plan_node/stream_changelog.rs +++ b/src/frontend/src/optimizer/plan_node/stream_changelog.rs @@ -72,9 +72,9 @@ impl_distill_by_unit!(StreamChangeLog, core, "StreamChangeLog"); impl StreamNode for StreamChangeLog { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { - PbNodeBody::Changelog(ChangeLogNode { + PbNodeBody::Changelog(Box::new(ChangeLogNode { need_op: self.core.need_op, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_dedup.rs b/src/frontend/src/optimizer/plan_node/stream_dedup.rs index d642d0f9e7ee0..689dd0432454d 100644 --- a/src/frontend/src/optimizer/plan_node/stream_dedup.rs +++ b/src/frontend/src/optimizer/plan_node/stream_dedup.rs @@ -92,7 +92,7 @@ impl StreamNode for StreamDedup { let table_catalog = self .infer_internal_table_catalog() .with_id(state.gen_table_id_wrapped()); - PbNodeBody::AppendOnlyDedup(DedupNode { + PbNodeBody::AppendOnlyDedup(Box::new(DedupNode { state_table: Some(table_catalog.to_internal_table_prost()), dedup_column_indices: self .core @@ -100,7 +100,7 @@ impl StreamNode for StreamDedup { .iter() .map(|idx| *idx as _) .collect_vec(), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs index 84592aee1829a..af9694471b3ea 100644 --- a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs @@ -159,7 +159,7 @@ impl TryToStreamPb for StreamDeltaJoin { // TODO: add a separate delta join node in proto, or move fragmenter to frontend so that we // don't need an intermediate representation. let eq_join_predicate = &self.eq_join_predicate; - Ok(NodeBody::DeltaIndexJoin(DeltaIndexJoinNode { + Ok(NodeBody::DeltaIndexJoin(Box::new(DeltaIndexJoinNode { join_type: self.core.join_type as i32, left_key: eq_join_predicate .left_eq_indexes() @@ -210,7 +210,7 @@ impl TryToStreamPb for StreamDeltaJoin { .collect(), }), output_indices: self.core.output_indices.iter().map(|&x| x as u32).collect(), - })) + }))) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_dml.rs b/src/frontend/src/optimizer/plan_node/stream_dml.rs index c69baf5ad53d9..72393159a4006 100644 --- a/src/frontend/src/optimizer/plan_node/stream_dml.rs +++ b/src/frontend/src/optimizer/plan_node/stream_dml.rs @@ -88,12 +88,12 @@ impl StreamNode for StreamDml { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - PbNodeBody::Dml(DmlNode { + PbNodeBody::Dml(Box::new(DmlNode { table_id: 0, // Meta will fill this table id. table_version_id: INITIAL_TABLE_VERSION_ID, // Meta will fill this version id. column_descs: self.column_descs.iter().map(Into::into).collect(), rate_limit: self.base.ctx().overwrite_options().dml_rate_limit, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs index 02fb426905c7e..1f704f5c9eb74 100644 --- a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs +++ b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs @@ -171,13 +171,13 @@ impl StreamNode for StreamDynamicFilter { let right_table = infer_right_internal_table_catalog(right.plan_base()) .with_id(state.gen_table_id_wrapped()); #[allow(deprecated)] - NodeBody::DynamicFilter(DynamicFilterNode { + NodeBody::DynamicFilter(Box::new(DynamicFilterNode { left_key: left_index as u32, condition, left_table: Some(left_table.to_internal_table_prost()), right_table: Some(right_table.to_internal_table_prost()), condition_always_relax: false, // deprecated - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs index 4d134df37799b..d27c1d417060e 100644 --- a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs +++ b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs @@ -155,12 +155,12 @@ impl StreamNode for StreamEowcOverWindow { .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(); - PbNodeBody::EowcOverWindow(EowcOverWindowNode { + PbNodeBody::EowcOverWindow(Box::new(EowcOverWindowNode { calls, partition_by, order_by, state_table: Some(state_table), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_exchange.rs b/src/frontend/src/optimizer/plan_node/stream_exchange.rs index d42f9a9392b41..28a41935d07a4 100644 --- a/src/frontend/src/optimizer/plan_node/stream_exchange.rs +++ b/src/frontend/src/optimizer/plan_node/stream_exchange.rs @@ -120,7 +120,7 @@ impl_plan_tree_node_for_unary! {StreamExchange} impl StreamNode for StreamExchange { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> NodeBody { - NodeBody::Exchange(ExchangeNode { + NodeBody::Exchange(Box::new(ExchangeNode { strategy: if self.no_shuffle { Some(DispatchStrategy { r#type: DispatcherType::NoShuffle as i32, @@ -144,7 +144,7 @@ impl StreamNode for StreamExchange { output_indices: (0..self.schema().len() as u32).collect(), }) }, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_expand.rs b/src/frontend/src/optimizer/plan_node/stream_expand.rs index fa0268a46fcf5..a6737fb22c360 100644 --- a/src/frontend/src/optimizer/plan_node/stream_expand.rs +++ b/src/frontend/src/optimizer/plan_node/stream_expand.rs @@ -79,13 +79,13 @@ impl_distill_by_unit!(StreamExpand, core, "StreamExpand"); impl StreamNode for StreamExpand { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { - PbNodeBody::Expand(ExpandNode { + PbNodeBody::Expand(Box::new(ExpandNode { column_subsets: self .column_subsets() .iter() .map(|subset| subset_to_protobuf(subset)) .collect(), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_filter.rs b/src/frontend/src/optimizer/plan_node/stream_filter.rs index 0a3126ffe7180..e3e6bb90a7f0b 100644 --- a/src/frontend/src/optimizer/plan_node/stream_filter.rs +++ b/src/frontend/src/optimizer/plan_node/stream_filter.rs @@ -69,9 +69,9 @@ impl_distill_by_unit!(StreamFilter, core, "StreamFilter"); impl StreamNode for StreamFilter { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { - PbNodeBody::Filter(FilterNode { + PbNodeBody::Filter(Box::new(FilterNode { search_condition: Some(ExprImpl::from(self.predicate().clone()).to_expr_proto()), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs index 9d3f46ca77cc3..f3ec2b280fef9 100644 --- a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs +++ b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs @@ -123,8 +123,8 @@ impl StreamNode for StreamFsFetch { secret_refs, } }); - NodeBody::StreamFsFetch(StreamFsFetchNode { + NodeBody::StreamFsFetch(Box::new(StreamFsFetchNode { node_inner: source_inner, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs b/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs index e2c795892e5f9..2b1f78f919c66 100644 --- a/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs +++ b/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs @@ -132,7 +132,7 @@ impl StreamNode for StreamGlobalApproxPercentile { .to_internal_table_prost(), ), }; - PbNodeBody::GlobalApproxPercentile(body) + PbNodeBody::GlobalApproxPercentile(Box::new(body)) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs index b9230270e634e..a539c0f2a1af2 100644 --- a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs +++ b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs @@ -129,9 +129,9 @@ impl StreamNode for StreamGroupTopN { order_by: self.topn_order().to_protobuf(), }; if self.input().append_only() { - PbNodeBody::AppendOnlyGroupTopN(group_topn_node) + PbNodeBody::AppendOnlyGroupTopN(Box::new(group_topn_node)) } else { - PbNodeBody::GroupTopN(group_topn_node) + PbNodeBody::GroupTopN(Box::new(group_topn_node)) } } } diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs index 66266a893c042..542b6784069f9 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs @@ -187,7 +187,7 @@ impl StreamNode for StreamHashAgg { self.core .infer_tables(&self.base, self.vnode_col_idx, self.window_col_idx); - PbNodeBody::HashAgg(HashAggNode { + PbNodeBody::HashAgg(Box::new(HashAggNode { group_key: self.group_key().to_vec_as_u32(), agg_calls: self .agg_calls() @@ -220,7 +220,7 @@ impl StreamNode for StreamHashAgg { row_count_index: self.row_count_idx as u32, emit_on_window_close: self.base.emit_on_window_close(), version: PbAggNodeVersion::Issue13465 as _, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs index 0d7863a247d9c..b330a9e24c6d8 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs @@ -359,7 +359,7 @@ impl StreamNode for StreamHashJoin { let null_safe_prost = self.eq_join_predicate.null_safes().into_iter().collect(); - NodeBody::HashJoin(HashJoinNode { + NodeBody::HashJoin(Box::new(HashJoinNode { join_type: self.core.join_type as i32, left_key: left_jk_indices_prost, right_key: right_jk_indices_prost, @@ -403,7 +403,7 @@ impl StreamNode for StreamHashJoin { right_deduped_input_pk_indices, output_indices: self.core.output_indices.iter().map(|&x| x as u32).collect(), is_append_only: self.is_append_only, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs index 4a50387c50be0..ee86b37fe35d7 100644 --- a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs +++ b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs @@ -104,7 +104,7 @@ impl_plan_tree_node_for_unary! {StreamHopWindow} impl StreamNode for StreamHopWindow { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { - PbNodeBody::HopWindow(HopWindowNode { + PbNodeBody::HopWindow(Box::new(HopWindowNode { time_col: self.core.time_col.index() as _, window_slide: Some(self.core.window_slide.into()), window_size: Some(self.core.window_size.into()), @@ -121,7 +121,7 @@ impl StreamNode for StreamHopWindow { .iter() .map(|x| x.to_expr_proto()) .collect(), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs b/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs index b5af9be49df05..c62c1f5d45d45 100644 --- a/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs +++ b/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs @@ -120,7 +120,7 @@ impl StreamNode for StreamLocalApproxPercentile { base, percentile_index, }; - PbNodeBody::LocalApproxPercentile(body) + PbNodeBody::LocalApproxPercentile(Box::new(body)) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs index 2793c84b1d555..fa912c7046985 100644 --- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs +++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs @@ -97,6 +97,7 @@ impl StreamMaterialize { } else { CreateType::Foreground }; + let table = Self::derive_table_catalog( input.clone(), name, @@ -136,15 +137,14 @@ impl StreamMaterialize { version_column_index: Option, pk_column_indices: Vec, row_id_index: Option, - version: Option, + version: TableVersion, retention_seconds: Option, - cdc_table_id: Option, webhook_info: Option, engine: Engine, ) -> Result { let input = Self::rewrite_input(input, user_distributed_by, TableType::Table)?; - let mut table = Self::derive_table_catalog( + let table = Self::derive_table_catalog( input.clone(), name, user_order_by, @@ -155,7 +155,7 @@ impl StreamMaterialize { Some(pk_column_indices), row_id_index, TableType::Table, - version, + Some(version), Cardinality::unknown(), // unknown cardinality for tables retention_seconds, CreateType::Foreground, @@ -163,8 +163,6 @@ impl StreamMaterialize { engine, )?; - table.cdc_table_id = cdc_table_id; - Ok(Self::new(input, table)) } @@ -385,7 +383,7 @@ impl StreamNode for StreamMaterialize { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - PbNodeBody::Materialize(MaterializeNode { + PbNodeBody::Materialize(Box::new(MaterializeNode { // We don't need table id for materialize node in frontend. The id will be generated on // meta catalog service. table_id: 0, @@ -396,7 +394,7 @@ impl StreamNode for StreamMaterialize { .map(ColumnOrder::to_protobuf) .collect(), table: Some(self.table().to_internal_table_prost()), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_now.rs b/src/frontend/src/optimizer/plan_node/stream_now.rs index 9ec80d15bac30..b8db60b11507b 100644 --- a/src/frontend/src/optimizer/plan_node/stream_now.rs +++ b/src/frontend/src/optimizer/plan_node/stream_now.rs @@ -81,7 +81,7 @@ impl StreamNode for StreamNow { let table_catalog = internal_table_catalog_builder .build(dist_keys, 0) .with_id(state.gen_table_id_wrapped()); - NodeBody::Now(PbNowNode { + NodeBody::Now(Box::new(PbNowNode { state_table: Some(table_catalog.to_internal_table_prost()), mode: Some(match &self.core.mode { Mode::UpdateCurrent => PbNowMode::UpdateCurrent(PbNowModeUpdateCurrent {}), @@ -93,7 +93,7 @@ impl StreamNode for StreamNow { interval: Some(Datum::Some((*interval).into()).to_protobuf()), }), }), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_over_window.rs index 6b0beaa9f99cc..405f25301f7b4 100644 --- a/src/frontend/src/optimizer/plan_node/stream_over_window.rs +++ b/src/frontend/src/optimizer/plan_node/stream_over_window.rs @@ -130,13 +130,13 @@ impl StreamNode for StreamOverWindow { .config() .streaming_over_window_cache_policy(); - PbNodeBody::OverWindow(OverWindowNode { + PbNodeBody::OverWindow(Box::new(OverWindowNode { calls, partition_by, order_by, state_table: Some(state_table), cache_policy: cache_policy.to_protobuf() as _, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_project.rs b/src/frontend/src/optimizer/plan_node/stream_project.rs index d6ac8af7b146c..92aefd429ad9c 100644 --- a/src/frontend/src/optimizer/plan_node/stream_project.rs +++ b/src/frontend/src/optimizer/plan_node/stream_project.rs @@ -159,13 +159,13 @@ impl StreamNode for StreamProject { .iter() .map(|(i, o)| (*i as u32, *o as u32)) .unzip(); - PbNodeBody::Project(ProjectNode { + PbNodeBody::Project(Box::new(ProjectNode { select_list: self.core.exprs.iter().map(|x| x.to_expr_proto()).collect(), watermark_input_cols, watermark_output_cols, nondecreasing_exprs: self.nondecreasing_exprs.iter().map(|i| *i as _).collect(), noop_update_hint: self.noop_update_hint, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_project_set.rs b/src/frontend/src/optimizer/plan_node/stream_project_set.rs index 5735c4b9d5644..5e75ba7d3cb56 100644 --- a/src/frontend/src/optimizer/plan_node/stream_project_set.rs +++ b/src/frontend/src/optimizer/plan_node/stream_project_set.rs @@ -111,7 +111,7 @@ impl StreamNode for StreamProjectSet { .iter() .map(|(i, o)| (*i as u32, *o as u32)) .unzip(); - PbNodeBody::ProjectSet(ProjectSetNode { + PbNodeBody::ProjectSet(Box::new(ProjectSetNode { select_list: self .core .select_list @@ -121,7 +121,7 @@ impl StreamNode for StreamProjectSet { watermark_input_cols, watermark_expr_indices, nondecreasing_exprs: self.nondecreasing_exprs.iter().map(|i| *i as _).collect(), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs index 36b96f4dad36e..3ac868dc224fb 100644 --- a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs +++ b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs @@ -83,9 +83,9 @@ impl StreamNode for StreamRowIdGen { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - PbNodeBody::RowIdGen(RowIdGenNode { + PbNodeBody::RowIdGen(Box::new(RowIdGenNode { row_id_index: self.row_id_index as _, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_row_merge.rs b/src/frontend/src/optimizer/plan_node/stream_row_merge.rs index 1295702135ffc..c065ffcc5a2a8 100644 --- a/src/frontend/src/optimizer/plan_node/stream_row_merge.rs +++ b/src/frontend/src/optimizer/plan_node/stream_row_merge.rs @@ -141,10 +141,10 @@ impl_plan_tree_node_for_binary! { StreamRowMerge } impl StreamNode for StreamRowMerge { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { - PbNodeBody::RowMerge(risingwave_pb::stream_plan::RowMergeNode { + PbNodeBody::RowMerge(Box::new(risingwave_pb::stream_plan::RowMergeNode { lhs_mapping: Some(self.lhs_mapping.to_protobuf()), rhs_mapping: Some(self.rhs_mapping.to_protobuf()), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs index f9f125654f402..66fe3424e06cf 100644 --- a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs @@ -113,7 +113,7 @@ impl StreamNode for StreamSimpleAgg { let (intermediate_state_table, agg_states, distinct_dedup_tables) = self.core.infer_tables(&self.base, None, None); - PbNodeBody::SimpleAgg(SimpleAggNode { + PbNodeBody::SimpleAgg(Box::new(SimpleAggNode { agg_calls: self .agg_calls() .iter() @@ -151,7 +151,7 @@ impl StreamNode for StreamSimpleAgg { row_count_index: self.row_count_idx as u32, version: PbAggNodeVersion::Issue13465 as _, must_output_per_barrier: self.must_output_per_barrier, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs index d9e0d116a61e5..1451a86ffb68f 100644 --- a/src/frontend/src/optimizer/plan_node/stream_sink.rs +++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs @@ -598,12 +598,12 @@ impl StreamNode for StreamSink { .infer_kv_log_store_table_catalog() .with_id(state.gen_table_id_wrapped()); - PbNodeBody::Sink(SinkNode { + PbNodeBody::Sink(Box::new(SinkNode { sink_desc: Some(self.sink_desc.to_proto()), table: Some(table.to_internal_table_prost()), log_store_type: self.log_store_type as i32, rate_limit: self.base.ctx().overwrite_options().sink_rate_limit, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_sort.rs b/src/frontend/src/optimizer/plan_node/stream_sort.rs index c4acd275f1236..3b5cbf7e5d85c 100644 --- a/src/frontend/src/optimizer/plan_node/stream_sort.rs +++ b/src/frontend/src/optimizer/plan_node/stream_sort.rs @@ -128,14 +128,14 @@ impl_plan_tree_node_for_unary! { StreamEowcSort } impl StreamNode for StreamEowcSort { fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - PbNodeBody::Sort(SortNode { + PbNodeBody::Sort(Box::new(SortNode { state_table: Some( self.infer_state_table() .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost(), ), sort_column_index: self.sort_column_index as _, - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_source.rs b/src/frontend/src/optimizer/plan_node/stream_source.rs index 909fa1e0d3009..b8e3ea6bf1834 100644 --- a/src/frontend/src/optimizer/plan_node/stream_source.rs +++ b/src/frontend/src/optimizer/plan_node/stream_source.rs @@ -96,7 +96,7 @@ impl StreamNode for StreamSource { secret_refs, } }); - PbNodeBody::Source(SourceNode { source_inner }) + PbNodeBody::Source(Box::new(SourceNode { source_inner })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs index f72361efbf03d..ab002a23d262e 100644 --- a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs +++ b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs @@ -158,7 +158,7 @@ impl StreamSourceScan { ..Default::default() }, ], - node_body: Some(PbNodeBody::SourceBackfill(backfill)), + node_body: Some(PbNodeBody::SourceBackfill(Box::new(backfill))), stream_key, operator_id: self.base.id().0 as u64, identity: self.distill_to_string(), diff --git a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs index edb9121baf595..43ae49d3bdaa4 100644 --- a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs +++ b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs @@ -84,7 +84,7 @@ impl_plan_tree_node_for_unary! { StreamStatelessSimpleAgg } impl StreamNode for StreamStatelessSimpleAgg { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody { use risingwave_pb::stream_plan::*; - PbNodeBody::StatelessSimpleAgg(SimpleAggNode { + PbNodeBody::StatelessSimpleAgg(Box::new(SimpleAggNode { agg_calls: self .agg_calls() .iter() @@ -103,7 +103,7 @@ impl StreamNode for StreamStatelessSimpleAgg { distinct_dedup_tables: Default::default(), version: AggNodeVersion::Issue13465 as _, must_output_per_barrier: false, // this is not used - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs index b75ec5b1c1d46..426d2add79f5d 100644 --- a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs +++ b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs @@ -310,7 +310,7 @@ impl StreamTableScan { None }; - let node_body = PbNodeBody::StreamScan(StreamScanNode { + let node_body = PbNodeBody::StreamScan(Box::new(StreamScanNode { table_id: self.core.table_desc.table_id.table_id, stream_scan_type: self.stream_scan_type as i32, // The column indices need to be forwarded to the downstream @@ -322,7 +322,7 @@ impl StreamTableScan { arrangement_table, rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit, ..Default::default() - }); + })); Ok(PbStreamNode { fields: self.schema().to_prost(), @@ -338,7 +338,7 @@ impl StreamTableScan { }, // Snapshot read PbStreamNode { - node_body: Some(PbNodeBody::BatchPlan(batch_plan_node)), + node_body: Some(PbNodeBody::BatchPlan(Box::new(batch_plan_node))), operator_id: self.batch_plan_id.0 as u64, identity: "BatchPlanNode".into(), fields: snapshot_schema, diff --git a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs index 390a141dcb385..b76c55a0b810c 100644 --- a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs +++ b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs @@ -246,7 +246,7 @@ impl TryToStreamPb for StreamTemporalJoin { .as_stream_table_scan() .expect("should be a stream table scan"); - Ok(NodeBody::TemporalJoin(TemporalJoinNode { + Ok(NodeBody::TemporalJoin(Box::new(TemporalJoinNode { join_type: self.core.join_type as i32, left_key: left_jk_indices_prost, right_key: right_jk_indices_prost, @@ -267,7 +267,7 @@ impl TryToStreamPb for StreamTemporalJoin { Some(memo_table.to_internal_table_prost()) }, is_nested_loop: self.is_nested_loop, - })) + }))) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_topn.rs b/src/frontend/src/optimizer/plan_node/stream_topn.rs index 80ca9141033c5..9109a76316884 100644 --- a/src/frontend/src/optimizer/plan_node/stream_topn.rs +++ b/src/frontend/src/optimizer/plan_node/stream_topn.rs @@ -112,9 +112,9 @@ impl StreamNode for StreamTopN { order_by: self.topn_order().to_protobuf(), }; if self.input().append_only() { - PbNodeBody::AppendOnlyTopN(topn_node) + PbNodeBody::AppendOnlyTopN(Box::new(topn_node)) } else { - PbNodeBody::TopN(topn_node) + PbNodeBody::TopN(Box::new(topn_node)) } } } diff --git a/src/frontend/src/optimizer/plan_node/stream_values.rs b/src/frontend/src/optimizer/plan_node/stream_values.rs index 0a71c208c32ee..eb42ecba57852 100644 --- a/src/frontend/src/optimizer/plan_node/stream_values.rs +++ b/src/frontend/src/optimizer/plan_node/stream_values.rs @@ -72,7 +72,7 @@ impl Distill for StreamValues { impl StreamNode for StreamValues { fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> ProstStreamNode { - ProstStreamNode::Values(ValuesNode { + ProstStreamNode::Values(Box::new(ValuesNode { tuples: self .logical .rows() @@ -86,7 +86,7 @@ impl StreamNode for StreamValues { .iter() .map(|f| f.to_prost()) .collect(), - }) + })) } } diff --git a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs index 63699998ea4b0..1d0935225abb0 100644 --- a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs +++ b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs @@ -148,12 +148,12 @@ impl StreamNode for StreamWatermarkFilter { let table = infer_internal_table_catalog(watermark_type); - PbNodeBody::WatermarkFilter(WatermarkFilterNode { + PbNodeBody::WatermarkFilter(Box::new(WatermarkFilterNode { watermark_descs: self.watermark_descs.clone(), tables: vec![table .with_id(state.gen_table_id_wrapped()) .to_internal_table_prost()], - }) + })) } } diff --git a/src/frontend/src/scheduler/distributed/stats.rs b/src/frontend/src/scheduler/distributed/stats.rs index 0cda386c6de88..506426f7fa144 100644 --- a/src/frontend/src/scheduler/distributed/stats.rs +++ b/src/frontend/src/scheduler/distributed/stats.rs @@ -14,16 +14,17 @@ use std::sync::LazyLock; -use prometheus::core::{AtomicI64, AtomicU64, GenericCounter, GenericGauge}; +use prometheus::core::{AtomicU64, GenericCounter}; use prometheus::{ exponential_buckets, histogram_opts, register_histogram_with_registry, - register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, Registry, + register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, IntGauge, + Registry, }; use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY; #[derive(Clone)] pub struct DistributedQueryMetrics { - pub running_query_num: GenericGauge, + pub running_query_num: IntGauge, pub rejected_query_counter: GenericCounter, pub completed_query_counter: GenericCounter, pub query_latency: Histogram, diff --git a/src/frontend/src/stream_fragmenter/mod.rs b/src/frontend/src/stream_fragmenter/mod.rs index f30b0abf5b4c4..b9447fd678444 100644 --- a/src/frontend/src/stream_fragmenter/mod.rs +++ b/src/frontend/src/stream_fragmenter/mod.rs @@ -439,9 +439,9 @@ fn build_fragment( let node = state.gen_no_op_stream_node(StreamNode { operator_id: no_shuffle_exchange_operator_id, identity: "StreamNoShuffleExchange".into(), - node_body: Some(NodeBody::Exchange(ExchangeNode { + node_body: Some(NodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(no_shuffle_strategy.clone()), - })), + }))), input: vec![], // Take reference's properties. diff --git a/src/frontend/src/stream_fragmenter/rewrite/delta_join.rs b/src/frontend/src/stream_fragmenter/rewrite/delta_join.rs index 8b6a2cb06c871..5c441a3aefeab 100644 --- a/src/frontend/src/stream_fragmenter/rewrite/delta_join.rs +++ b/src/frontend/src/stream_fragmenter/rewrite/delta_join.rs @@ -35,11 +35,11 @@ fn build_no_shuffle_exchange_for_delta_join( identity: "NO SHUFFLE Exchange (Lookup and Merge)".into(), fields: upstream.fields.clone(), stream_key: upstream.stream_key.clone(), - node_body: Some(NodeBody::Exchange(ExchangeNode { + node_body: Some(NodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(dispatch_no_shuffle( (0..(upstream.fields.len() as u32)).collect(), )), - })), + }))), input: vec![], append_only: upstream.append_only, } @@ -55,12 +55,12 @@ fn build_consistent_hash_shuffle_exchange_for_delta_join( identity: "HASH Exchange (Lookup and Merge)".into(), fields: upstream.fields.clone(), stream_key: upstream.stream_key.clone(), - node_body: Some(NodeBody::Exchange(ExchangeNode { + node_body: Some(NodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(dispatch_consistent_hash_shuffle( dist_key_indices, (0..(upstream.fields.len() as u32)).collect(), )), - })), + }))), input: vec![], append_only: upstream.append_only, } @@ -97,7 +97,7 @@ fn build_lookup_for_delta_join( identity: "Lookup".into(), fields: output_fields, stream_key: output_stream_key, - node_body: Some(NodeBody::Lookup(lookup_node)), + node_body: Some(NodeBody::Lookup(Box::new(lookup_node))), input: vec![ exchange_node_arrangement.clone(), exchange_node_stream.clone(), @@ -281,7 +281,9 @@ fn build_delta_join_inner( identity: "Union".into(), fields: node.fields.clone(), stream_key: node.stream_key.clone(), - node_body: Some(NodeBody::LookupUnion(LookupUnionNode { order: vec![1, 0] })), + node_body: Some(NodeBody::LookupUnion(Box::new(LookupUnionNode { + order: vec![1, 0], + }))), input: vec![exchange_l0m.clone(), exchange_l1m.clone()], append_only: node.append_only, }; diff --git a/src/frontend/src/utils/condition.rs b/src/frontend/src/utils/condition.rs index e497ba821864a..fb5fd07c1fc07 100644 --- a/src/frontend/src/utils/condition.rs +++ b/src/frontend/src/utils/condition.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::Ordering; use std::collections::{BTreeMap, HashSet}; use std::fmt::{self, Debug}; use std::ops::Bound; @@ -24,6 +25,7 @@ use risingwave_common::catalog::{Schema, TableDesc}; use risingwave_common::types::{DataType, DefaultOrd, ScalarImpl}; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::scan_range::{is_full_range, ScanRange}; +use risingwave_common::util::sort_util::{cmp_rows, OrderType}; use crate::error::Result; use crate::expr::{ @@ -299,7 +301,7 @@ impl Condition { table_desc: Rc, max_split_range_gap: u64, disjunctions: Vec, - ) -> Result, Self)>> { + ) -> Result, bool)>> { let disjunctions_result: Result, Self)>> = disjunctions .into_iter() .map(|x| { @@ -352,9 +354,154 @@ impl Condition { } } - Ok(Some((non_overlap_scan_ranges, Condition::true_cond()))) + Ok(Some((non_overlap_scan_ranges, false))) } else { - Ok(None) + let mut scan_ranges = vec![]; + for (scan_ranges_chunk, _) in disjunctions_result { + if scan_ranges_chunk.is_empty() { + // full scan range + return Ok(None); + } + + scan_ranges.extend(scan_ranges_chunk); + } + + let order_types = table_desc + .pk + .iter() + .cloned() + .map(|x| { + if x.order_type.is_descending() { + x.order_type.reverse() + } else { + x.order_type + } + }) + .collect_vec(); + scan_ranges.sort_by(|left, right| { + let (left_start, _left_end) = &left.convert_to_range(); + let (right_start, _right_end) = &right.convert_to_range(); + + let left_start_vec = match &left_start { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + let right_start_vec = match &right_start { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + _ => &vec![], + }; + + if left_start_vec.is_empty() && right_start_vec.is_empty() { + return Ordering::Less; + } + + if left_start_vec.is_empty() { + return Ordering::Less; + } + + if right_start_vec.is_empty() { + return Ordering::Greater; + } + + let cmp_column_len = left_start_vec.len().min(right_start_vec.len()); + cmp_rows( + &left_start_vec[0..cmp_column_len], + &right_start_vec[0..cmp_column_len], + &order_types[0..cmp_column_len], + ) + }); + + if scan_ranges.is_empty() { + return Ok(None); + } + + if scan_ranges.len() == 1 { + return Ok(Some((scan_ranges, true))); + } + + let mut output_scan_ranges: Vec = vec![]; + output_scan_ranges.push(scan_ranges[0].clone()); + let mut idx = 1; + loop { + if idx >= scan_ranges.len() { + break; + } + + let scan_range_left = output_scan_ranges.last_mut().unwrap(); + let scan_range_right = &scan_ranges[idx]; + + if scan_range_left.eq_conds == scan_range_right.eq_conds { + // range merge + + if !ScanRange::is_overlap(scan_range_left, scan_range_right, &order_types) { + // not merge + output_scan_ranges.push(scan_range_right.clone()); + idx += 1; + continue; + } + + // merge range + fn merge_bound( + left_scan_range: &Bound>>, + right_scan_range: &Bound>>, + order_types: &[OrderType], + left_bound: bool, + ) -> Bound>> { + let left_scan_range = match left_scan_range { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + Bound::Unbounded => return Bound::Unbounded, + }; + + let right_scan_range = match right_scan_range { + Bound::Included(vec) | Bound::Excluded(vec) => vec, + Bound::Unbounded => return Bound::Unbounded, + }; + + let cmp_len = left_scan_range.len().min(right_scan_range.len()); + + let cmp = cmp_rows( + &left_scan_range[..cmp_len], + &right_scan_range[..cmp_len], + &order_types[..cmp_len], + ); + + let bound = { + if (cmp.is_le() && left_bound) || (cmp.is_ge() && !left_bound) { + left_scan_range.to_vec() + } else { + right_scan_range.to_vec() + } + }; + + // Included Bound just for convenience, the correctness will be guaranteed by the upper level filter. + Bound::Included(bound) + } + + scan_range_left.range.0 = merge_bound( + &scan_range_left.range.0, + &scan_range_right.range.0, + &order_types, + true, + ); + + scan_range_left.range.1 = merge_bound( + &scan_range_left.range.1, + &scan_range_right.range.1, + &order_types, + false, + ); + + if scan_range_left.is_full_table_scan() { + return Ok(None); + } + } else { + output_scan_ranges.push(scan_range_right.clone()); + } + + idx += 1; + } + + Ok(Some((output_scan_ranges, true))) } } @@ -415,28 +562,43 @@ impl Condition { || matches!(func_type, ExprType::GreaterThan)) { let mut pk_struct = vec![]; + let mut order_type = None; let mut all_added = true; let mut iter = row_left_inputs.iter().zip_eq_fast(right_iter); - for i in 0..table_desc.order_column_indices().len() { + for column_order in &table_desc.pk { if let Some((left_expr, right_expr)) = iter.next() { - if left_expr.as_input_ref().unwrap().index != i { + if left_expr.as_input_ref().unwrap().index != column_order.column_index { all_added = false; break; } + match order_type { + Some(o) => { + if o != column_order.order_type { + all_added = false; + break; + } + } + None => order_type = Some(column_order.order_type), + } pk_struct.push(right_expr.clone()); } } + // Here it is necessary to determine whether all of row is included in the `ScanRanges`, if so, the data for eq is not needed if !pk_struct.is_empty() { - let scan_range = ScanRange { - eq_conds: vec![], - range: match func_type { - ExprType::GreaterThan => (Bound::Excluded(pk_struct), Bound::Unbounded), - ExprType::LessThan => (Bound::Unbounded, Bound::Excluded(pk_struct)), - _ => unreachable!(), - }, - }; if !all_added { + let scan_range = ScanRange { + eq_conds: vec![], + range: match func_type { + ExprType::GreaterThan => { + (Bound::Included(pk_struct), Bound::Unbounded) + } + ExprType::LessThan => { + (Bound::Unbounded, Bound::Included(pk_struct)) + } + _ => unreachable!(), + }, + }; return Ok(Some(( vec![scan_range], Condition { @@ -444,6 +606,18 @@ impl Condition { }, ))); } else { + let scan_range = ScanRange { + eq_conds: vec![], + range: match func_type { + ExprType::GreaterThan => { + (Bound::Excluded(pk_struct), Bound::Unbounded) + } + ExprType::LessThan => { + (Bound::Unbounded, Bound::Excluded(pk_struct)) + } + _ => unreachable!(), + }, + }; return Ok(Some(( vec![scan_range], Condition { @@ -470,12 +644,18 @@ impl Condition { // It's an OR. if self.conjunctions.len() == 1 { if let Some(disjunctions) = self.conjunctions[0].as_or_disjunctions() { - if let Some((scan_ranges, other_condition)) = Self::disjunctions_to_scan_ranges( - table_desc, - max_split_range_gap, - disjunctions, - )? { - return Ok((scan_ranges, other_condition)); + if let Some((scan_ranges, maintaining_condition)) = + Self::disjunctions_to_scan_ranges( + table_desc, + max_split_range_gap, + disjunctions, + )? + { + if maintaining_condition { + return Ok((scan_ranges, self)); + } else { + return Ok((scan_ranges, Condition::true_cond())); + } } else { return Ok((vec![], self)); } diff --git a/src/java_binding/Cargo.toml b/src/java_binding/Cargo.toml index 0966b700a713f..36facf1790abd 100644 --- a/src/java_binding/Cargo.toml +++ b/src/java_binding/Cargo.toml @@ -3,12 +3,6 @@ name = "risingwave_java_binding" version = "0.1.0" edition = "2021" -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" bytes = "1" diff --git a/src/jni_core/Cargo.toml b/src/jni_core/Cargo.toml index 868c79e7427ec..d182e19045997 100644 --- a/src/jni_core/Cargo.toml +++ b/src/jni_core/Cargo.toml @@ -3,29 +3,18 @@ name = "risingwave_jni_core" version = "0.1.0" edition = "2021" -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" bytes = "1" cfg-or-panic = "0.2" chrono = { version = "0.4", default-features = false } -foyer ={ workspace = true } fs-err = "3" futures = { version = "0.3", default-features = false, features = ["alloc"] } -itertools = { workspace = true } jni = { version = "0.21.1", features = ["invocation"] } paste = "1" prost = { workspace = true } risingwave_common = { workspace = true } risingwave_pb = { workspace = true } -rw_futures_util = { workspace = true } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" thiserror = "1" thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ @@ -41,8 +30,6 @@ tracing = "0.1" [dev-dependencies] expect-test = "1" -risingwave_expr = { workspace = true } -risingwave_hummock_sdk = { workspace = true } [lints] workspace = true diff --git a/src/license/Cargo.toml b/src/license/Cargo.toml index b435747467e21..3c0c2e8f612ff 100644 --- a/src/license/Cargo.toml +++ b/src/license/Cargo.toml @@ -8,12 +8,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] jsonbb = { workspace = true } jsonwebtoken = "9" diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml index 16a303609699b..582903c99a00a 100644 --- a/src/meta/Cargo.toml +++ b/src/meta/Cargo.toml @@ -7,21 +7,12 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" arc-swap = "1" assert_matches = "1" async-trait = "0.1" -aws-config = { workspace = true } -aws-sdk-ec2 = { workspace = true } base64-url = { version = "3.0.0" } -bincode = "1.3" bytes = { version = "1", features = ["serde"] } chrono = "0.4" clap = { workspace = true } @@ -32,17 +23,12 @@ educe = "0.6" either = "1" enum-as-inner = "0.6" fail = "0.5" -flate2 = "1" -function_name = "0.3.0" futures = { version = "0.3", default-features = false, features = ["alloc"] } hex = "0.4" http = "1" itertools = { workspace = true } jsonbb = { workspace = true } maplit = "1.0.2" -memcomparable = { version = "0.2" } -mime_guess = "2" -moka = { version = "0.12.3", features = ["future"] } notify = { version = "7", default-features = false, features = [ "macos_fsevent", ] } @@ -89,7 +75,6 @@ tokio-stream = { workspace = true } tonic = { workspace = true } tower = { version = "0.5", features = ["util", "load-shed"] } tracing = "0.1" -url = "2" uuid = { version = "1", features = ["v4"] } [target.'cfg(not(madsim))'.dependencies] diff --git a/src/meta/model/Cargo.toml b/src/meta/model/Cargo.toml index 991becc820642..530a67da3b9f5 100644 --- a/src/meta/model/Cargo.toml +++ b/src/meta/model/Cargo.toml @@ -7,17 +7,10 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] prost = { workspace = true } risingwave_common = { workspace = true } -risingwave_hummock_sdk = { workspace = true } risingwave_pb = { workspace = true } sea-orm = { workspace = true } serde = { version = "1.0.196", features = ["derive"] } -serde_json = "1.0.113" +serde_json = "1" diff --git a/src/meta/model/migration/Cargo.toml b/src/meta/model/migration/Cargo.toml index 9d9fbd5292d25..725e028cabdc7 100644 --- a/src/meta/model/migration/Cargo.toml +++ b/src/meta/model/migration/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] async-std = { version = "1", features = ["attributes", "tokio1"] } easy-ext = "1" diff --git a/src/meta/node/Cargo.toml b/src/meta/node/Cargo.toml index dc011842c3cd3..f3f13d546ec06 100644 --- a/src/meta/node/Cargo.toml +++ b/src/meta/node/Cargo.toml @@ -7,20 +7,10 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] -anyhow = "1" clap = { workspace = true } educe = "0.6" -either = "1" -futures = { version = "0.3", default-features = false, features = ["alloc"] } hex = "0.4" -itertools = { workspace = true } otlp-embedded = { workspace = true } prometheus-http-query = "0.8" redact = "0.1.5" @@ -29,12 +19,10 @@ risingwave_common = { workspace = true } risingwave_common_heap_profiling = { workspace = true } risingwave_common_service = { workspace = true } risingwave_meta = { workspace = true } -risingwave_meta_model_migration = { workspace = true } risingwave_meta_service = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } sea-orm = { workspace = true } -serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml index 53c3708da0e12..b3fbe5e8bcb78 100644 --- a/src/meta/service/Cargo.toml +++ b/src/meta/service/Cargo.toml @@ -7,19 +7,11 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" async-trait = "0.1" -either = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = { workspace = true } -prost = { workspace = true } rand = { workspace = true } regex = "1" risingwave_common = { workspace = true } @@ -30,7 +22,6 @@ risingwave_meta_model = { workspace = true } risingwave_pb = { workspace = true } sea-orm = { workspace = true } serde_json = "1" -sync-point = { path = "../../utils/sync-point" } thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", diff --git a/src/meta/src/barrier/checkpoint/control.rs b/src/meta/src/barrier/checkpoint/control.rs index 28ee5aa0bfe32..4a22ff3a0ac2f 100644 --- a/src/meta/src/barrier/checkpoint/control.rs +++ b/src/meta/src/barrier/checkpoint/control.rs @@ -50,6 +50,7 @@ use crate::barrier::{ }; use crate::manager::ActiveStreamingWorkerNodes; use crate::rpc::metrics::GLOBAL_META_METRICS; +use crate::stream::fill_snapshot_backfill_epoch; use crate::{MetaError, MetaResult}; #[derive(Default)] @@ -695,8 +696,10 @@ impl DatabaseCheckpointControl { progress_epoch, creating_job .snapshot_backfill_info - .upstream_mv_table_ids - .clone(), + .upstream_mv_table_id_to_backfill_epoch + .keys() + .cloned() + .collect(), ), ) }) @@ -977,7 +980,7 @@ impl DatabaseCheckpointControl { if let Some(Command::CreateStreamingJob { job_type: CreateStreamingJobType::SnapshotBackfill(snapshot_backfill_info), info, - }) = &command + }) = &mut command { if self.state.paused_reason().is_some() { warn!("cannot create streaming job with snapshot backfill when paused"); @@ -989,18 +992,44 @@ impl DatabaseCheckpointControl { } return Ok(()); } + // set snapshot epoch of upstream table for snapshot backfill + for snapshot_backfill_epoch in snapshot_backfill_info + .upstream_mv_table_id_to_backfill_epoch + .values_mut() + { + assert!( + snapshot_backfill_epoch + .replace(barrier_info.prev_epoch()) + .is_none(), + "must not set previously" + ); + } + for stream_actor in info + .stream_job_fragments + .fragments + .values_mut() + .flat_map(|fragment| fragment.actors.iter_mut()) + { + fill_snapshot_backfill_epoch( + stream_actor.nodes.as_mut().expect("should exist"), + &snapshot_backfill_info.upstream_mv_table_id_to_backfill_epoch, + )?; + } + let info = info.clone(); + let job_id = info.stream_job_fragments.stream_job_id(); + let snapshot_backfill_info = snapshot_backfill_info.clone(); let mutation = command .as_ref() .expect("checked Some") .to_mutation(None) .expect("should have some mutation in `CreateStreamingJob` command"); - let job_id = info.stream_job_fragments.stream_job_id(); + control_stream_manager.add_partial_graph(self.database_id, Some(job_id))?; self.creating_streaming_job_controls.insert( job_id, CreatingStreamingJobControl::new( - info.clone(), - snapshot_backfill_info.clone(), + info, + snapshot_backfill_info, barrier_info.prev_epoch(), hummock_version_stats, mutation, diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs index 3ded5c438a670..b42986d354a60 100644 --- a/src/meta/src/barrier/command.rs +++ b/src/meta/src/barrier/command.rs @@ -227,7 +227,10 @@ impl CreateStreamingJobCommandInfo { #[derive(Debug, Clone)] pub struct SnapshotBackfillInfo { - pub upstream_mv_table_ids: HashSet, + /// `table_id` -> `Some(snapshot_backfill_epoch)` + /// The `snapshot_backfill_epoch` should be None at the beginning, and be filled + /// by global barrier worker when handling the command. + pub upstream_mv_table_id_to_backfill_epoch: HashMap>, } #[derive(Debug, Clone)] @@ -697,8 +700,8 @@ impl Command { job_type { snapshot_backfill_info - .upstream_mv_table_ids - .iter() + .upstream_mv_table_id_to_backfill_epoch + .keys() .map(|table_id| SubscriptionUpstreamInfo { subscriber_id: table_fragments.stream_job_id().table_id, upstream_mv_table_id: table_id.table_id, @@ -747,12 +750,13 @@ impl Command { info: jobs_to_merge .iter() .flat_map(|(table_id, (backfill_info, _))| { - backfill_info.upstream_mv_table_ids.iter().map( - move |upstream_table_id| SubscriptionUpstreamInfo { + backfill_info + .upstream_mv_table_id_to_backfill_epoch + .keys() + .map(move |upstream_table_id| SubscriptionUpstreamInfo { subscriber_id: table_id.table_id, upstream_mv_table_id: upstream_table_id.table_id, - }, - ) + }) }) .collect(), })) diff --git a/src/meta/src/barrier/context/context_impl.rs b/src/meta/src/barrier/context/context_impl.rs index f1306a99d7823..a045708b72dd4 100644 --- a/src/meta/src/barrier/context/context_impl.rs +++ b/src/meta/src/barrier/context/context_impl.rs @@ -19,6 +19,7 @@ use risingwave_common::catalog::DatabaseId; use risingwave_pb::common::WorkerNode; use risingwave_pb::hummock::HummockVersionStats; use risingwave_pb::meta::PausedReason; +use risingwave_pb::stream_plan::PbFragmentTypeFlag; use risingwave_pb::stream_service::streaming_control_stream_request::PbInitRequest; use risingwave_pb::stream_service::WaitEpochCommitRequest; use risingwave_rpc_client::StreamingControlHandle; @@ -178,8 +179,57 @@ impl CommandContext { .unregister_table_ids(unregistered_state_table_ids.iter().cloned()) .await?; } - Command::CreateStreamingJob { info, job_type } => { + let mut fragment_replacements = None; + let mut dropped_actors = None; + match job_type { + CreateStreamingJobType::SinkIntoTable( + replace_plan @ ReplaceStreamJobPlan { + new_fragments, + dispatchers, + init_split_assignment, + .. + }, + ) => { + barrier_manager_context + .metadata_manager + .catalog_controller + .post_collect_job_fragments( + new_fragments.stream_job_id().table_id as _, + new_fragments.actor_ids(), + dispatchers.clone(), + init_split_assignment, + ) + .await?; + fragment_replacements = Some(replace_plan.fragment_replacements()); + dropped_actors = Some(replace_plan.dropped_actors()); + } + CreateStreamingJobType::Normal => {} + CreateStreamingJobType::SnapshotBackfill(snapshot_backfill_info) => { + barrier_manager_context + .metadata_manager + .catalog_controller + .fill_snapshot_backfill_epoch( + info.stream_job_fragments.fragments.iter().filter_map( + |(fragment_id, fragment)| { + if (fragment.fragment_type_mask + & PbFragmentTypeFlag::SnapshotBackfillStreamScan as u32) + != 0 + { + Some(*fragment_id as _) + } else { + None + } + }, + ), + &snapshot_backfill_info.upstream_mv_table_id_to_backfill_epoch, + ) + .await? + } + } + + // Do `post_collect_job_fragments` of the original streaming job in the end, so that in any previous failure, + // we won't mark the job as `Creating`, and then the job will be later clean by the recovery triggered by the returned error. let CreateStreamingJobCommandInfo { stream_job_fragments, dispatchers, @@ -197,31 +247,6 @@ impl CommandContext { ) .await?; - let mut fragment_replacements = None; - let mut dropped_actors = None; - if let CreateStreamingJobType::SinkIntoTable( - replace_plan @ ReplaceStreamJobPlan { - new_fragments, - dispatchers, - init_split_assignment, - .. - }, - ) = job_type - { - barrier_manager_context - .metadata_manager - .catalog_controller - .post_collect_job_fragments( - new_fragments.stream_job_id().table_id as _, - new_fragments.actor_ids(), - dispatchers.clone(), - init_split_assignment, - ) - .await?; - fragment_replacements = Some(replace_plan.fragment_replacements()); - dropped_actors = Some(replace_plan.dropped_actors()); - } - // Extract the fragments that include source operators. let source_fragments = stream_job_fragments.stream_source_fragments(); let backfill_fragments = stream_job_fragments.source_backfill_fragments()?; diff --git a/src/meta/src/controller/fragment.rs b/src/meta/src/controller/fragment.rs index a9a536f0b394d..6e71835618dfb 100644 --- a/src/meta/src/controller/fragment.rs +++ b/src/meta/src/controller/fragment.rs @@ -1293,6 +1293,34 @@ impl CatalogController { Ok(()) } + pub async fn fill_snapshot_backfill_epoch( + &self, + fragment_ids: impl Iterator, + upstream_mv_snapshot_epoch: &HashMap>, + ) -> MetaResult<()> { + let inner = self.inner.write().await; + let txn = inner.db.begin().await?; + for fragment_id in fragment_ids { + let fragment = Fragment::find_by_id(fragment_id) + .one(&txn) + .await? + .context(format!("fragment {} not found", fragment_id))?; + let mut node = fragment.stream_node.to_protobuf(); + if crate::stream::fill_snapshot_backfill_epoch(&mut node, upstream_mv_snapshot_epoch)? { + let node = StreamNode::from(&node); + Fragment::update(fragment::ActiveModel { + fragment_id: Set(fragment_id), + stream_node: Set(node), + ..Default::default() + }) + .exec(&txn) + .await?; + } + } + txn.commit().await?; + Ok(()) + } + /// Get the actor ids of the fragment with `fragment_id` with `Running` status. pub async fn get_running_actors_of_fragment( &self, @@ -1639,11 +1667,11 @@ mod tests { let mut input = vec![]; for (upstream_fragment_id, upstream_actor_ids) in actor_upstream_actor_ids { input.push(PbStreamNode { - node_body: Some(PbNodeBody::Merge(MergeNode { + node_body: Some(PbNodeBody::Merge(Box::new(MergeNode { upstream_actor_id: upstream_actor_ids.iter().map(|id| *id as _).collect(), upstream_fragment_id: *upstream_fragment_id as _, ..Default::default() - })), + }))), ..Default::default() }); } diff --git a/src/meta/src/dashboard/mod.rs b/src/meta/src/dashboard/mod.rs index ed52378c700fc..22ebf74acb47e 100644 --- a/src/meta/src/dashboard/mod.rs +++ b/src/meta/src/dashboard/mod.rs @@ -49,6 +49,7 @@ pub struct DashboardService { pub type Service = Arc; pub(super) mod handlers { + use std::cmp::min; use std::collections::HashMap; use anyhow::Context; @@ -488,7 +489,28 @@ pub(super) mod handlers { let result = result .map_err(|_| anyhow!("Failed to get back pressure")) .map_err(err)?; + // TODO(eric): aggregate back_pressure_infos here all.back_pressure_infos.extend(result.back_pressure_infos); + + // Aggregate fragment_stats + for (fragment_id, fragment_stats) in result.fragment_stats { + if let Some(s) = all.fragment_stats.get_mut(&fragment_id) { + s.actor_count += fragment_stats.actor_count; + s.current_epoch = min(s.current_epoch, fragment_stats.current_epoch); + } else { + all.fragment_stats.insert(fragment_id, fragment_stats); + } + } + + // Aggregate relation_stats + for (relation_id, relation_stats) in result.relation_stats { + if let Some(s) = all.relation_stats.get_mut(&relation_id) { + s.actor_count += relation_stats.actor_count; + s.current_epoch = min(s.current_epoch, relation_stats.current_epoch); + } else { + all.relation_stats.insert(relation_id, relation_stats); + } + } } Ok(all.into()) diff --git a/src/meta/src/hummock/manager/time_travel.rs b/src/meta/src/hummock/manager/time_travel.rs index 3e333ecafb94c..acd851b3b2781 100644 --- a/src/meta/src/hummock/manager/time_travel.rs +++ b/src/meta/src/hummock/manager/time_travel.rs @@ -484,6 +484,8 @@ impl HummockManager { .on_conflict_do_nothing() .exec(txn) .await?; + // Return early to skip persisting delta. + return Ok(version_sst_ids); } let written = write_sstable_infos( delta.newly_added_sst_infos().filter(|s| { diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index b3c8b47b29af2..89200b4b63d6c 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -871,7 +871,7 @@ impl DdlController { format!("ProjectExecutor(from sink {})", sink_id); } - *merge_node = MergeNode { + **merge_node = MergeNode { upstream_actor_id: sink_actor_ids.clone(), upstream_fragment_id, upstream_dispatcher_type: DispatcherType::Hash as _, diff --git a/src/meta/src/stream/stream_graph.rs b/src/meta/src/stream/stream_graph.rs index 3268dcb4d16c1..c1e7763e94b88 100644 --- a/src/meta/src/stream/stream_graph.rs +++ b/src/meta/src/stream/stream_graph.rs @@ -18,5 +18,7 @@ mod id; mod schedule; pub use actor::{ActorGraphBuildResult, ActorGraphBuilder}; -pub use fragment::{CompleteStreamFragmentGraph, StreamFragmentGraph}; +pub use fragment::{ + fill_snapshot_backfill_epoch, CompleteStreamFragmentGraph, StreamFragmentGraph, +}; pub use schedule::Locations; diff --git a/src/meta/src/stream/stream_graph/actor.rs b/src/meta/src/stream/stream_graph/actor.rs index 093f45b27958d..3d290605a38a9 100644 --- a/src/meta/src/stream/stream_graph/actor.rs +++ b/src/meta/src/stream/stream_graph/actor.rs @@ -151,12 +151,12 @@ impl ActorBuilder { }]; Ok(StreamNode { - node_body: Some(NodeBody::Merge(MergeNode { + node_body: Some(NodeBody::Merge(Box::new(MergeNode { upstream_actor_id: upstreams.actors.as_global_ids(), upstream_fragment_id: upstreams.fragment_id.as_global_id(), upstream_dispatcher_type: exchange.get_strategy()?.r#type, fields: stream_node.get_fields().clone(), - })), + }))), identity: "MergeExecutor".to_owned(), ..stream_node.clone() }) @@ -197,12 +197,12 @@ impl ActorBuilder { let input = vec![ // Fill the merge node body with correct upstream info. StreamNode { - node_body: Some(NodeBody::Merge(MergeNode { + node_body: Some(NodeBody::Merge(Box::new(MergeNode { upstream_actor_id, upstream_fragment_id: upstreams.fragment_id.as_global_id(), upstream_dispatcher_type, fields: merge_node.fields.clone(), - })), + }))), ..merge_node.clone() }, batch_plan_node.clone(), @@ -246,12 +246,12 @@ impl ActorBuilder { let input = vec![ // Fill the merge node body with correct upstream info. StreamNode { - node_body: Some(NodeBody::Merge(MergeNode { + node_body: Some(NodeBody::Merge(Box::new(MergeNode { upstream_actor_id, upstream_fragment_id: upstreams.fragment_id.as_global_id(), upstream_dispatcher_type: DispatcherType::NoShuffle as _, fields: merge_node.fields.clone(), - })), + }))), ..merge_node.clone() }, ]; diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs index 21103fe1460c1..1203dbfe93d0a 100644 --- a/src/meta/src/stream/stream_graph/fragment.rs +++ b/src/meta/src/stream/stream_graph/fragment.rs @@ -27,7 +27,9 @@ use risingwave_common::catalog::{ use risingwave_common::hash::VnodeCount; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::stream_graph_visitor; -use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont; +use risingwave_common::util::stream_graph_visitor::{ + visit_stream_node_cont, visit_stream_node_cont_mut, +}; use risingwave_meta_model::WorkerId; use risingwave_pb::catalog::Table; use risingwave_pb::ddl_service::TableJobType; @@ -38,7 +40,7 @@ use risingwave_pb::stream_plan::stream_fragment_graph::{ use risingwave_pb::stream_plan::stream_node::NodeBody; use risingwave_pb::stream_plan::{ DispatchStrategy, DispatcherType, FragmentTypeFlag, StreamActor, - StreamFragmentGraph as StreamFragmentGraphProto, StreamScanNode, StreamScanType, + StreamFragmentGraph as StreamFragmentGraphProto, StreamNode, StreamScanNode, StreamScanType, }; use crate::barrier::SnapshotBackfillInfo; @@ -602,8 +604,8 @@ impl StreamFragmentGraph { match (prev_snapshot_backfill_info, is_snapshot_backfill) { (Some(prev_snapshot_backfill_info), true) => { prev_snapshot_backfill_info - .upstream_mv_table_ids - .insert(TableId::new(stream_scan.table_id)); + .upstream_mv_table_id_to_backfill_epoch + .insert(TableId::new(stream_scan.table_id), None); true } (None, false) => true, @@ -617,14 +619,14 @@ impl StreamFragmentGraph { prev_stream_scan = Some(( if is_snapshot_backfill { Some(SnapshotBackfillInfo { - upstream_mv_table_ids: HashSet::from_iter([TableId::new( - stream_scan.table_id, - )]), + upstream_mv_table_id_to_backfill_epoch: HashMap::from_iter( + [(TableId::new(stream_scan.table_id), None)], + ), }) } else { None }, - stream_scan.clone(), + *stream_scan.clone(), )); true } @@ -642,6 +644,44 @@ impl StreamFragmentGraph { } } +/// Fill snapshot epoch for `StreamScanNode` of `SnapshotBackfill`. +/// Return `true` when has change applied. +pub fn fill_snapshot_backfill_epoch( + node: &mut StreamNode, + upstream_mv_table_snapshot_epoch: &HashMap>, +) -> MetaResult { + let mut result = Ok(()); + let mut applied = false; + visit_stream_node_cont_mut(node, |node| { + if let Some(NodeBody::StreamScan(stream_scan)) = node.node_body.as_mut() + && stream_scan.stream_scan_type == StreamScanType::SnapshotBackfill as i32 + { + result = try { + let table_id = TableId::new(stream_scan.table_id); + let snapshot_epoch = upstream_mv_table_snapshot_epoch + .get(&table_id) + .ok_or_else(|| anyhow!("upstream table id not covered: {}", table_id))? + .ok_or_else(|| anyhow!("upstream table id not set: {}", table_id))?; + if let Some(prev_snapshot_epoch) = + stream_scan.snapshot_backfill_epoch.replace(snapshot_epoch) + { + Err(anyhow!( + "snapshot backfill epoch set again: {} {} {}", + table_id, + prev_snapshot_epoch, + snapshot_epoch + ))?; + } + applied = true; + }; + result.is_ok() + } else { + true + } + }); + result.map(|_| applied) +} + static EMPTY_HASHMAP: LazyLock> = LazyLock::new(HashMap::new); diff --git a/src/meta/src/stream/test_fragmenter.rs b/src/meta/src/stream/test_fragmenter.rs index ce844a6c764cb..3d64f378fcf5f 100644 --- a/src/meta/src/stream/test_fragmenter.rs +++ b/src/meta/src/stream/test_fragmenter.rs @@ -206,14 +206,14 @@ fn make_stream_fragments() -> Vec { }) .collect_vec(); let source_node = StreamNode { - node_body: Some(NodeBody::Source(SourceNode { + node_body: Some(NodeBody::Source(Box::new(SourceNode { source_inner: Some(StreamSource { source_id: 1, state_table: Some(make_source_internal_table(0)), columns, ..Default::default() }), - })), + }))), stream_key: vec![2], ..Default::default() }; @@ -228,13 +228,13 @@ fn make_stream_fragments() -> Vec { // exchange node let exchange_node = StreamNode { - node_body: Some(NodeBody::Exchange(ExchangeNode { + node_body: Some(NodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(DispatchStrategy { r#type: DispatcherType::Hash as i32, dist_key_indices: vec![0], output_indices: vec![0, 1, 2], }), - })), + }))), fields: vec![ make_field(TypeName::Int32), make_field(TypeName::Int32), @@ -252,7 +252,7 @@ fn make_stream_fragments() -> Vec { children: vec![make_inputref(0), make_inputref(1)], }; let filter_node = StreamNode { - node_body: Some(NodeBody::Filter(FilterNode { + node_body: Some(NodeBody::Filter(Box::new(FilterNode { search_condition: Some(ExprNode { function_type: GreaterThan as i32, return_type: Some(DataType { @@ -261,7 +261,7 @@ fn make_stream_fragments() -> Vec { }), rex_node: Some(RexNode::FuncCall(function_call)), }), - })), + }))), fields: vec![], // TODO: fill this later input: vec![exchange_node], stream_key: vec![0, 1], @@ -272,14 +272,14 @@ fn make_stream_fragments() -> Vec { // simple agg node let simple_agg_node = StreamNode { - node_body: Some(NodeBody::SimpleAgg(SimpleAggNode { + node_body: Some(NodeBody::SimpleAgg(Box::new(SimpleAggNode { agg_calls: vec![make_sum_aggcall(0), make_sum_aggcall(1)], distribution_key: Default::default(), is_append_only: false, agg_call_states: vec![make_agg_call_result_state(), make_agg_call_result_state()], intermediate_state_table: Some(make_empty_table(1)), ..Default::default() - })), + }))), input: vec![filter_node], fields: vec![], // TODO: fill this later stream_key: vec![0, 1], @@ -299,12 +299,12 @@ fn make_stream_fragments() -> Vec { // exchange node let exchange_node_1 = StreamNode { - node_body: Some(NodeBody::Exchange(ExchangeNode { + node_body: Some(NodeBody::Exchange(Box::new(ExchangeNode { strategy: Some(DispatchStrategy { r#type: DispatcherType::Simple as i32, ..Default::default() }), - })), + }))), fields: vec![make_field(TypeName::Int64), make_field(TypeName::Int64)], input: vec![], stream_key: vec![0, 1], @@ -315,14 +315,14 @@ fn make_stream_fragments() -> Vec { // agg node let simple_agg_node_1 = StreamNode { - node_body: Some(NodeBody::SimpleAgg(SimpleAggNode { + node_body: Some(NodeBody::SimpleAgg(Box::new(SimpleAggNode { agg_calls: vec![make_sum_aggcall(0), make_sum_aggcall(1)], distribution_key: Default::default(), is_append_only: false, agg_call_states: vec![make_agg_call_result_state(), make_agg_call_result_state()], intermediate_state_table: Some(make_empty_table(2)), ..Default::default() - })), + }))), fields: vec![], // TODO: fill this later input: vec![exchange_node_1], stream_key: vec![0, 1], @@ -336,7 +336,7 @@ fn make_stream_fragments() -> Vec { children: vec![make_inputref(0), make_inputref(1)], }; let project_node = StreamNode { - node_body: Some(NodeBody::Project(ProjectNode { + node_body: Some(NodeBody::Project(Box::new(ProjectNode { select_list: vec![ ExprNode { rex_node: Some(RexNode::FuncCall(function_call_1)), @@ -350,7 +350,7 @@ fn make_stream_fragments() -> Vec { make_inputref(1), ], ..Default::default() - })), + }))), fields: vec![], // TODO: fill this later input: vec![simple_agg_node_1], stream_key: vec![1, 2], @@ -363,11 +363,11 @@ fn make_stream_fragments() -> Vec { let mview_node = StreamNode { input: vec![project_node], stream_key: vec![], - node_body: Some(NodeBody::Materialize(MaterializeNode { + node_body: Some(NodeBody::Materialize(Box::new(MaterializeNode { table_id: 1, table: Some(make_materialize_table(888)), column_orders: vec![make_column_order(1), make_column_order(2)], - })), + }))), fields: vec![], // TODO: fill this later operator_id: 7, identity: "MaterializeExecutor".to_owned(), diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index 54bc6b4524f66..0e7d213a72321 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -15,6 +15,7 @@ workspace = true async-trait = "0.1" await-tree = { workspace = true } aws-config = { workspace = true } +# add the dependency explicitly to enable hardcoded-credentials feature aws-credential-types = { workspace = true } aws-sdk-s3 = { version = "0.5", package = "madsim-aws-sdk-s3" } aws-smithy-http = { workspace = true } @@ -23,12 +24,10 @@ aws-smithy-runtime-api = { workspace = true } aws-smithy-types = { workspace = true } bytes = { version = "1", features = ["serde"] } crc32fast = "1" -either = "1" fail = "0.5" futures = { version = "0.3", default-features = false, features = ["alloc"] } hyper = { version = "0.14", features = ["tcp", "client"] } # TODO(http-bump): required by aws sdk hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] } -hyper-tls = "0.5.0" itertools = { workspace = true } madsim = "0.2.31" opendal = { workspace = true, features = [ @@ -56,11 +55,9 @@ tracing = "0.1" # This crate is excluded from hakari (see hakari.toml) after hdfs is introduced...## [target.'cfg(not(madsim))'.dependencies] # workspace-hack = { path = "../workspace-hack" } # -# [package.metadata.cargo-machete] -# ignored = ["workspace-hack"] # -# [package.metadata.cargo-udeps.ignore] -# normal = ["workspace-hack"] +# +# # [features] # hdfs-backend = ["opendal/services-hdfs", "dep:risingwave_jni_core"] diff --git a/src/prost/Cargo.toml b/src/prost/Cargo.toml index 230a654e0fcee..c9fcb69dafd9b 100644 --- a/src/prost/Cargo.toml +++ b/src/prost/Cargo.toml @@ -21,6 +21,9 @@ tonic = { workspace = true } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../workspace-hack" } +[dev-dependencies] +static_assertions = "1" + [build-dependencies] fs-err = "3.0" pbjson-build = "0.7" @@ -28,11 +31,5 @@ prost-build = { workspace = true } tonic-build = { workspace = true } walkdir = "2" -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [lints] workspace = true diff --git a/src/prost/build.rs b/src/prost/build.rs index e906c47efbd55..554fc1d67b446 100644 --- a/src/prost/build.rs +++ b/src/prost/build.rs @@ -112,6 +112,55 @@ fn main() -> Result<(), Box> { "#[derive(::enum_as_inner::EnumAsInner)]", ) .btree_map(btree_map_paths) + // node body is a very large enum, so we box it to avoid stack overflow. + // TODO: ideally we should box all enum variants automatically https://github.com/tokio-rs/prost/issues/1209 + .boxed(".stream_plan.StreamNode.node_body.source") + .boxed(".stream_plan.StreamNode.node_body.project") + .boxed(".stream_plan.StreamNode.node_body.filter") + .boxed(".stream_plan.StreamNode.node_body.materialize") + .boxed(".stream_plan.StreamNode.node_body.stateless_simple_agg") + .boxed(".stream_plan.StreamNode.node_body.simple_agg") + .boxed(".stream_plan.StreamNode.node_body.hash_agg") + .boxed(".stream_plan.StreamNode.node_body.append_only_top_n") + .boxed(".stream_plan.StreamNode.node_body.hash_join") + .boxed(".stream_plan.StreamNode.node_body.top_n") + .boxed(".stream_plan.StreamNode.node_body.hop_window") + .boxed(".stream_plan.StreamNode.node_body.merge") + .boxed(".stream_plan.StreamNode.node_body.exchange") + .boxed(".stream_plan.StreamNode.node_body.stream_scan") + .boxed(".stream_plan.StreamNode.node_body.batch_plan") + .boxed(".stream_plan.StreamNode.node_body.lookup") + .boxed(".stream_plan.StreamNode.node_body.arrange") + .boxed(".stream_plan.StreamNode.node_body.lookup_union") + .boxed(".stream_plan.StreamNode.node_body.delta_index_join") + .boxed(".stream_plan.StreamNode.node_body.sink") + .boxed(".stream_plan.StreamNode.node_body.expand") + .boxed(".stream_plan.StreamNode.node_body.dynamic_filter") + .boxed(".stream_plan.StreamNode.node_body.project_set") + .boxed(".stream_plan.StreamNode.node_body.group_top_n") + .boxed(".stream_plan.StreamNode.node_body.sort") + .boxed(".stream_plan.StreamNode.node_body.watermark_filter") + .boxed(".stream_plan.StreamNode.node_body.dml") + .boxed(".stream_plan.StreamNode.node_body.row_id_gen") + .boxed(".stream_plan.StreamNode.node_body.now") + .boxed(".stream_plan.StreamNode.node_body.append_only_group_top_n") + .boxed(".stream_plan.StreamNode.node_body.temporal_join") + .boxed(".stream_plan.StreamNode.node_body.barrier_recv") + .boxed(".stream_plan.StreamNode.node_body.values") + .boxed(".stream_plan.StreamNode.node_body.append_only_dedup") + .boxed(".stream_plan.StreamNode.node_body.eowc_over_window") + .boxed(".stream_plan.StreamNode.node_body.over_window") + .boxed(".stream_plan.StreamNode.node_body.stream_fs_fetch") + .boxed(".stream_plan.StreamNode.node_body.stream_cdc_scan") + .boxed(".stream_plan.StreamNode.node_body.cdc_filter") + .boxed(".stream_plan.StreamNode.node_body.source_backfill") + .boxed(".stream_plan.StreamNode.node_body.changelog") + .boxed(".stream_plan.StreamNode.node_body.local_approx_percentile") + .boxed(".stream_plan.StreamNode.node_body.global_approx_percentile") + .boxed(".stream_plan.StreamNode.node_body.row_merge") + .boxed(".stream_plan.StreamNode.node_body.as_of_join") + // `Udf` is 248 bytes, while 2nd largest field is 32 bytes. + .boxed(".expr.ExprNode.rex_node.udf") // Eq + Hash are for plan nodes to do common sub-plan detection. // The requirement is from Source node -> SourceCatalog -> WatermarkDesc -> expr .type_attribute("catalog.WatermarkDesc", "#[derive(Eq, Hash)]") diff --git a/src/prost/helpers/Cargo.toml b/src/prost/helpers/Cargo.toml index c78ac3f2a8ece..177b1b97dadaa 100644 --- a/src/prost/helpers/Cargo.toml +++ b/src/prost/helpers/Cargo.toml @@ -11,11 +11,5 @@ proc-macro2 = { version = "1", default-features = false } quote = "1" syn = "2" -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [lints] workspace = true diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs index 8fa584522354d..294d5d9514bcf 100644 --- a/src/prost/src/lib.rs +++ b/src/prost/src/lib.rs @@ -13,8 +13,15 @@ // limitations under the License. // for derived code of `Message` -#![expect(clippy::all)] #![expect(clippy::doc_markdown)] +#![expect(clippy::upper_case_acronyms)] +#![expect(clippy::needless_lifetimes)] +// For tonic::transport::Endpoint::connect +#![expect(clippy::disallowed_methods)] +#![expect(clippy::enum_variant_names)] +#![expect(clippy::module_inception)] +// FIXME: This should be fixed!!! https://github.com/risingwavelabs/risingwave/issues/19906 +#![expect(clippy::large_enum_variant)] use std::collections::HashMap; use std::str::FromStr; @@ -496,11 +503,12 @@ impl std::fmt::Debug for plan_common::ColumnDesc { s.field("additional_column_type", additional_column_type); } s.field("version", version); - if let Some(AdditionalColumn { column_type }) = additional_column { + if let Some(AdditionalColumn { + column_type: Some(column_type), + }) = additional_column + { // AdditionalColumn { None } means a normal column - if let Some(column_type) = column_type { - s.field("additional_column", &column_type); - } + s.field("additional_column", &column_type); } if let Some(generated_or_default_column) = generated_or_default_column { s.field("generated_or_default_column", &generated_or_default_column); @@ -513,11 +521,14 @@ impl std::fmt::Debug for plan_common::ColumnDesc { mod tests { use crate::data::{data_type, DataType}; use crate::plan_common::Field; + use crate::stream_plan::stream_node::NodeBody; #[test] fn test_getter() { - let mut data_type: DataType = DataType::default(); - data_type.is_nullable = true; + let data_type: DataType = DataType { + is_nullable: true, + ..Default::default() + }; let field = Field { data_type: Some(data_type), name: "".to_owned(), @@ -551,4 +562,12 @@ mod tests { }; assert!(!new_data_type.is_nullable); } + + #[test] + fn test_size() { + use static_assertions::const_assert_eq; + // box all fields in NodeBody to avoid large_enum_variant + // see https://github.com/risingwavelabs/risingwave/issues/19910 + const_assert_eq!(std::mem::size_of::(), 16); + } } diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml index 2c415d9f5da78..3334ea6bd4162 100644 --- a/src/risedevtool/Cargo.toml +++ b/src/risedevtool/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = { version = "1", features = ["backtrace"] } chrono = { version = "0.4", default-features = false, features = [ @@ -32,7 +26,6 @@ redis = "0.25" regex = "1" reqwest = { version = "0.12.2", features = ["blocking"] } serde = { version = "1", features = ["derive"] } -serde_json = "1" serde_with = "3" serde_yaml = "0.9" sqlx = { workspace = true, features = ["any"] } diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml index 1da4027c5110c..d10a7d8d766d7 100644 --- a/src/rpc_client/Cargo.toml +++ b/src/rpc_client/Cargo.toml @@ -7,22 +7,13 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" async-trait = "0.1" easy-ext = "1" either = "1.13.0" futures = { version = "0.3", default-features = false, features = ["alloc"] } -h2 = "0.4.6" # https://github.com/risingwavelabs/risingwave/issues/18039 http = "1" -hyper = "1" -itertools = { workspace = true } lru = { workspace = true } moka = { version = "0.12.0", features = ["future"] } paste = "1" @@ -48,7 +39,6 @@ tokio-stream = { workspace = true } tonic = { workspace = true } tower = "0.5" tracing = "0.1" -url = "2.5.0" [dev-dependencies] risingwave_hummock_sdk = { workspace = true } diff --git a/src/sqlparser/Cargo.toml b/src/sqlparser/Cargo.toml index 7638d27208beb..18c1068122d34 100644 --- a/src/sqlparser/Cargo.toml +++ b/src/sqlparser/Cargo.toml @@ -15,12 +15,6 @@ path = "src/lib.rs" default = ["std"] std = [] -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] itertools = { workspace = true } serde = { version = "1.0", features = ["derive"], optional = true } diff --git a/src/sqlparser/fuzz/Cargo.toml b/src/sqlparser/fuzz/Cargo.toml index 24ebb8e6ba7fd..c031295f78fec 100644 --- a/src/sqlparser/fuzz/Cargo.toml +++ b/src/sqlparser/fuzz/Cargo.toml @@ -4,11 +4,9 @@ version = "0.1.0" edition = "2018" publish = false -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] + + [dependencies] honggfuzz = "0.5.54" diff --git a/src/sqlparser/sqlparser_bench/Cargo.toml b/src/sqlparser/sqlparser_bench/Cargo.toml index f28d7ef75e2a2..a654ad5a6fb94 100644 --- a/src/sqlparser/sqlparser_bench/Cargo.toml +++ b/src/sqlparser/sqlparser_bench/Cargo.toml @@ -4,11 +4,9 @@ version = "0.1.0" authors = ["Dandandan "] edition = "2018" -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] + + [dependencies] risingwave_sqlparser = { workspace = true } diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index c0fce4a2d4780..ec525ddcc5951 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -2839,7 +2839,7 @@ impl fmt::Display for EmitMode { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum OnConflict { UpdateFull, diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index a5ccd017bdd75..39dc955964105 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -7,22 +7,13 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] -ahash = "0.8.7" anyhow = "1" arc-swap = "1" async-trait = "0.1" auto_enums = { workspace = true } await-tree = { workspace = true } bytes = { version = "1", features = ["serde"] } -crossbeam = "0.8.2" -dashmap = { version = "6", default-features = false } dyn-clone = "1.0.14" either = "1" enum-as-inner = "0.6" diff --git a/src/storage/backup/Cargo.toml b/src/storage/backup/Cargo.toml index 23a2c99567c48..f9b88044eb69e 100644 --- a/src/storage/backup/Cargo.toml +++ b/src/storage/backup/Cargo.toml @@ -7,16 +7,9 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" async-trait = "0.1" -bincode = "1.3" bytes = { version = "1", features = ["serde"] } itertools = { workspace = true } parking_lot = { workspace = true } diff --git a/src/storage/compactor/Cargo.toml b/src/storage/compactor/Cargo.toml index ed8f421ee6384..d94da125ca3ec 100644 --- a/src/storage/compactor/Cargo.toml +++ b/src/storage/compactor/Cargo.toml @@ -8,18 +8,11 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] async-trait = "0.1" await-tree = { workspace = true } clap = { workspace = true } jsonbb = { workspace = true } -parking_lot = { workspace = true } prost = { workspace = true } risingwave_common = { workspace = true } risingwave_common_heap_profiling = { workspace = true } diff --git a/src/storage/hummock_sdk/Cargo.toml b/src/storage/hummock_sdk/Cargo.toml index c6dff5afb32d9..8c911f12b5a91 100644 --- a/src/storage/hummock_sdk/Cargo.toml +++ b/src/storage/hummock_sdk/Cargo.toml @@ -7,25 +7,16 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack", "num-traits"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] bincode = { version = "=2.0.0-rc.3", features = ["serde"] } bytes = "1" -easy-ext = "1" hex = "0.4" itertools = { workspace = true } parse-display = "0.10" -prost = { workspace = true } risingwave_common = { workspace = true } risingwave_common_estimate_size = { workspace = true } risingwave_pb = { workspace = true } serde = { version = "1", features = ["derive"] } -serde_bytes = "0.11" tracing = "0.1" [target.'cfg(not(madsim))'.dependencies] diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs index 7f4b4dafb1bf7..f0bd4582752be 100644 --- a/src/storage/hummock_sdk/src/version.rs +++ b/src/storage/hummock_sdk/src/version.rs @@ -271,6 +271,21 @@ impl HummockVersion { .values() .map(|table_watermark| table_watermark.estimated_encode_len()) .sum::() + + self + .table_change_log + .values() + .map(|c| { + c.0.iter() + .map(|l| { + l.old_value + .iter() + .chain(l.new_value.iter()) + .map(|s| s.estimated_encode_len()) + .sum::() + }) + .sum::() + }) + .sum::() } } diff --git a/src/storage/hummock_test/Cargo.toml b/src/storage/hummock_test/Cargo.toml index da96b5883462a..45a5ce62c5ba1 100644 --- a/src/storage/hummock_test/Cargo.toml +++ b/src/storage/hummock_test/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] async-trait = "0.1" bytes = { version = "1" } @@ -34,7 +28,7 @@ risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_storage = { workspace = true, features = ["test"] } -serde = { version = "1", features = ["derive"] } +serial_test = { version = "3.2", optional = true } tokio = { version = "0.2", package = "madsim-tokio" } [target.'cfg(not(madsim))'.dependencies] @@ -51,12 +45,11 @@ futures = { version = "0.3", default-features = false, features = [ futures-async-stream = "0.2.9" risingwave_hummock_sdk = { workspace = true } risingwave_test_runner = { workspace = true } -serial_test = "3.2" sync-point = { path = "../../utils/sync-point" } [features] failpoints = ["risingwave_storage/failpoints"] -sync_point = ["sync-point/sync_point"] +sync_point = ["sync-point/sync_point", "serial_test"] test = [] [[bench]] diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index a2da11bee7159..912affeeae200 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -24,9 +24,9 @@ use await_tree::InstrumentAwait; use futures::FutureExt; use itertools::Itertools; use parking_lot::RwLock; -use prometheus::core::{AtomicU64, GenericGauge}; use prometheus::{Histogram, IntGauge}; use risingwave_common::catalog::TableId; +use risingwave_common::metrics::UintGauge; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo; use risingwave_hummock_sdk::{HummockEpoch, SyncResult}; use tokio::spawn; @@ -62,14 +62,11 @@ pub struct BufferTracker { flush_threshold: usize, min_batch_flush_size: usize, global_buffer: Arc, - global_upload_task_size: GenericGauge, + global_upload_task_size: UintGauge, } impl BufferTracker { - pub fn from_storage_opts( - config: &StorageOpts, - global_upload_task_size: GenericGauge, - ) -> Self { + pub fn from_storage_opts(config: &StorageOpts, global_upload_task_size: UintGauge) -> Self { let capacity = config.shared_buffer_capacity_mb * (1 << 20); let flush_threshold = (capacity as f32 * config.shared_buffer_flush_ratio) as usize; let shared_buffer_min_batch_flush_size = @@ -93,7 +90,7 @@ impl BufferTracker { Self::new( usize::MAX, flush_threshold, - GenericGauge::new("test", "test").unwrap(), + UintGauge::new("test", "test").unwrap(), min_batch_flush_size, ) } @@ -101,7 +98,7 @@ impl BufferTracker { fn new( capacity: usize, flush_threshold: usize, - global_upload_task_size: GenericGauge, + global_upload_task_size: UintGauge, min_batch_flush_size: usize, ) -> Self { assert!(capacity >= flush_threshold); @@ -116,7 +113,7 @@ impl BufferTracker { pub fn for_test() -> Self { Self::from_storage_opts( &StorageOpts::default(), - GenericGauge::new("test", "test").unwrap(), + UintGauge::new("test", "test").unwrap(), ) } @@ -128,7 +125,7 @@ impl BufferTracker { &self.global_buffer } - pub fn global_upload_task_size(&self) -> &GenericGauge { + pub fn global_upload_task_size(&self) -> &UintGauge { &self.global_upload_task_size } diff --git a/src/storage/src/hummock/event_handler/uploader/mod.rs b/src/storage/src/hummock/event_handler/uploader/mod.rs index 26a8fc436383f..4499aaa9700ea 100644 --- a/src/storage/src/hummock/event_handler/uploader/mod.rs +++ b/src/storage/src/hummock/event_handler/uploader/mod.rs @@ -28,10 +28,10 @@ use std::task::{ready, Context, Poll}; use futures::FutureExt; use itertools::Itertools; use more_asserts::assert_gt; -use prometheus::core::{AtomicU64, GenericGauge}; use prometheus::{HistogramTimer, IntGauge}; use risingwave_common::bitmap::BitmapBuilder; use risingwave_common::catalog::TableId; +use risingwave_common::metrics::UintGauge; use risingwave_common::must_match; use risingwave_hummock_sdk::table_watermark::{ TableWatermarks, VnodeWatermark, WatermarkDirection, WatermarkSerdeType, @@ -112,14 +112,14 @@ mod uploader_imm { use std::fmt::Formatter; use std::ops::Deref; - use prometheus::core::{AtomicU64, GenericGauge}; + use risingwave_common::metrics::UintGauge; use crate::hummock::event_handler::uploader::UploaderContext; use crate::mem_table::ImmutableMemtable; pub(super) struct UploaderImm { inner: ImmutableMemtable, - size_guard: GenericGauge, + size_guard: UintGauge, } impl UploaderImm { @@ -137,7 +137,7 @@ mod uploader_imm { pub(super) fn for_test(imm: ImmutableMemtable) -> Self { Self { inner: imm, - size_guard: GenericGauge::new("test", "test").unwrap(), + size_guard: UintGauge::new("test", "test").unwrap(), } } } @@ -175,7 +175,7 @@ struct UploadingTask { join_handle: JoinHandle>, task_info: UploadTaskInfo, spawn_upload_task: SpawnUploadTask, - task_size_guard: GenericGauge, + task_size_guard: UintGauge, task_count_guard: IntGauge, } @@ -481,6 +481,7 @@ struct LocalInstanceUnsyncData { sealed_data: VecDeque, // newer data comes first flushing_imms: VecDeque, + is_destroyed: bool, } impl LocalInstanceUnsyncData { @@ -491,10 +492,12 @@ impl LocalInstanceUnsyncData { current_epoch_data: Some(LocalInstanceEpochData::new(init_epoch)), sealed_data: VecDeque::new(), flushing_imms: Default::default(), + is_destroyed: false, } } fn add_imm(&mut self, imm: UploaderImm) { + assert!(!self.is_destroyed); assert_eq!(self.table_id, imm.table_id); self.current_epoch_data .as_mut() @@ -503,6 +506,7 @@ impl LocalInstanceUnsyncData { } fn local_seal_epoch(&mut self, next_epoch: HummockEpoch) -> HummockEpoch { + assert!(!self.is_destroyed); let data = self .current_epoch_data .as_mut() @@ -626,6 +630,10 @@ impl LocalInstanceUnsyncData { } } } + + fn is_finished(&self) -> bool { + self.is_destroyed && self.sealed_data.is_empty() + } } struct TableUnsyncData { @@ -903,18 +911,19 @@ impl UnsyncData { } } - fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) -> Option { - if let Some(table_id) = self.instance_table_id.remove(&instance_id) { + fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) { + if let Some(table_id) = self.instance_table_id.get(&instance_id) { debug!(instance_id, "destroy instance"); - let table_data = self.table_data.get_mut(&table_id).expect("should exist"); - assert!(table_data.instance_data.remove(&instance_id).is_some()); - if table_data.is_empty() { - Some(self.table_data.remove(&table_id).expect("should exist")) - } else { - None - } - } else { - None + let table_data = self.table_data.get_mut(table_id).expect("should exist"); + let instance_data = table_data + .instance_data + .get_mut(&instance_id) + .expect("should exist"); + assert!( + !instance_data.is_destroyed, + "cannot destroy an instance for twice" + ); + instance_data.is_destroyed = true; } } @@ -932,9 +941,20 @@ impl UnsyncData { } } assert!( - table_unsync_data.instance_data.is_empty(), + table_unsync_data + .instance_data + .values() + .all(|instance| instance.is_destroyed), "should be clear when dropping the read version instance" ); + for instance_id in table_unsync_data.instance_data.keys() { + assert_eq!( + *table_id, + self.instance_table_id + .remove(instance_id) + .expect("should exist") + ); + } } } debug_assert!(self @@ -1011,6 +1031,18 @@ impl UploaderData { flush_payload.insert(instance_id, payload); } } + table_data.instance_data.retain(|instance_id, data| { + // remove the finished instances + if data.is_finished() { + assert_eq!( + self.unsync_data.instance_table_id.remove(instance_id), + Some(*table_id) + ); + false + } else { + true + } + }); if let Some((direction, watermarks, watermark_type)) = table_watermarks { Self::add_table_watermarks( &mut all_table_watermarks, @@ -1429,28 +1461,7 @@ impl HummockUploader { let UploaderState::Working(data) = &mut self.state else { return; }; - if let Some(removed_table_data) = data.unsync_data.may_destroy_instance(instance_id) { - data.task_manager.remove_table_spill_tasks( - removed_table_data.table_id, - removed_table_data - .spill_tasks - .into_values() - .flat_map(|task_ids| task_ids.into_iter()) - .filter(|task_id| { - if let Some((_, table_ids)) = data.unsync_data.spilled_data.get_mut(task_id) - { - assert!(table_ids.remove(&removed_table_data.table_id)); - if table_ids.is_empty() { - data.unsync_data.spilled_data.remove(task_id); - } - false - } else { - true - } - }), - ) - } - data.check_upload_task_consistency(); + data.unsync_data.may_destroy_instance(instance_id); } pub(crate) fn min_uncommitted_sst_id(&self) -> Option { @@ -1800,6 +1811,65 @@ pub(crate) mod tests { ); } + #[tokio::test] + async fn test_uploader_destroy_instance_before_sync() { + let mut uploader = test_uploader(dummy_success_upload_future); + let epoch1 = INITIAL_EPOCH.next_epoch(); + uploader.start_epochs_for_test([epoch1]); + let imm = gen_imm(epoch1).await; + uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1); + uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm.clone()); + uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1); + uploader.may_destroy_instance(TEST_LOCAL_INSTANCE_ID); + + let (sync_tx, sync_rx) = oneshot::channel(); + uploader.start_single_epoch_sync(epoch1, sync_tx, HashSet::from_iter([TEST_TABLE_ID])); + assert_eq!(epoch1 as HummockEpoch, uploader.test_max_syncing_epoch()); + assert_eq!(1, uploader.data().syncing_data.len()); + let (_, syncing_data) = uploader.data().syncing_data.first_key_value().unwrap(); + assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_table_epochs[0].0); + assert!(syncing_data.uploaded.is_empty()); + assert!(!syncing_data.remaining_uploading_tasks.is_empty()); + + let staging_sst = uploader.next_uploaded_sst().await; + assert_eq!(&vec![epoch1], staging_sst.epochs()); + assert_eq!( + &HashMap::from_iter([(TEST_LOCAL_INSTANCE_ID, vec![imm.batch_id()])]), + staging_sst.imm_ids() + ); + assert_eq!( + &dummy_success_upload_output().new_value_ssts, + staging_sst.sstable_infos() + ); + + match sync_rx.await { + Ok(Ok(data)) => { + let SyncedData { + uploaded_ssts, + table_watermarks, + } = data; + assert_eq!(1, uploaded_ssts.len()); + let staging_sst = &uploaded_ssts[0]; + assert_eq!(&vec![epoch1], staging_sst.epochs()); + assert_eq!( + &HashMap::from_iter([(TEST_LOCAL_INSTANCE_ID, vec![imm.batch_id()])]), + staging_sst.imm_ids() + ); + assert_eq!( + &dummy_success_upload_output().new_value_ssts, + staging_sst.sstable_infos() + ); + assert!(table_watermarks.is_empty()); + } + _ => unreachable!(), + }; + assert!(!uploader + .data() + .unsync_data + .table_data + .contains_key(&TEST_TABLE_ID)); + } + #[tokio::test] async fn test_empty_uploader_sync() { let mut uploader = test_uploader(dummy_success_upload_future); diff --git a/src/storage/src/hummock/event_handler/uploader/task_manager.rs b/src/storage/src/hummock/event_handler/uploader/task_manager.rs index fd53fae1db322..c23aa648a7222 100644 --- a/src/storage/src/hummock/event_handler/uploader/task_manager.rs +++ b/src/storage/src/hummock/event_handler/uploader/task_manager.rs @@ -136,24 +136,6 @@ impl TaskManager { ) } - pub(super) fn remove_table_spill_tasks( - &mut self, - table_id: TableId, - task_ids: impl Iterator, - ) { - for task_id in task_ids { - let entry = self.tasks.get_mut(&task_id).expect("should exist"); - let empty = must_match!(&mut entry.status, UploadingTaskStatus::Spilling(table_ids) => { - assert!(table_ids.remove(&table_id)); - table_ids.is_empty() - }); - if empty { - let task = self.tasks.remove(&task_id).expect("should exist").task; - task.join_handle.abort(); - } - } - } - pub(super) fn sync( &mut self, context: &UploaderContext, diff --git a/src/storage/src/monitor/hummock_state_store_metrics.rs b/src/storage/src/monitor/hummock_state_store_metrics.rs index ced2723cafea0..acdc960f0f36c 100644 --- a/src/storage/src/monitor/hummock_state_store_metrics.rs +++ b/src/storage/src/monitor/hummock_state_store_metrics.rs @@ -14,7 +14,7 @@ use std::sync::{Arc, OnceLock}; -use prometheus::core::{AtomicU64, Collector, Desc, GenericCounter, GenericGauge}; +use prometheus::core::{AtomicU64, Collector, Desc, GenericCounter}; use prometheus::{ exponential_buckets, histogram_opts, proto, register_histogram_vec_with_registry, register_histogram_with_registry, register_int_counter_vec_with_registry, @@ -23,7 +23,7 @@ use prometheus::{ use risingwave_common::config::MetricLevel; use risingwave_common::metrics::{ RelabeledCounterVec, RelabeledGuardedHistogramVec, RelabeledGuardedIntCounterVec, - RelabeledHistogramVec, RelabeledMetricVec, + RelabeledHistogramVec, RelabeledMetricVec, UintGauge, }; use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY; use risingwave_common::{ @@ -73,9 +73,9 @@ pub struct HummockStateStoreMetrics { pub spill_task_size_from_sealed: GenericCounter, // uploading task - pub uploader_uploading_task_size: GenericGauge, + pub uploader_uploading_task_size: UintGauge, pub uploader_uploading_task_count: IntGauge, - pub uploader_imm_size: GenericGauge, + pub uploader_imm_size: UintGauge, pub uploader_upload_task_latency: Histogram, pub uploader_syncing_epoch_count: IntGauge, pub uploader_wait_poll_latency: Histogram, @@ -312,7 +312,7 @@ impl HummockStateStoreMetrics { ) .unwrap(); - let uploader_uploading_task_size = GenericGauge::new( + let uploader_uploading_task_size = UintGauge::new( "state_store_uploader_uploading_task_size", "Total size of uploader uploading tasks", ) @@ -328,7 +328,7 @@ impl HummockStateStoreMetrics { ) .unwrap(); - let uploader_imm_size = GenericGauge::new( + let uploader_imm_size = UintGauge::new( "state_store_uploader_imm_size", "Total size of imms tracked by uploader", ) diff --git a/src/stream/Cargo.toml b/src/stream/Cargo.toml index 8a0061ab99ea9..0061174fbbd7b 100644 --- a/src/stream/Cargo.toml +++ b/src/stream/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" arc-swap = "1" @@ -25,6 +19,7 @@ await-tree = { workspace = true } bytes = "1" cfg-if = "1" delta_btree_map = { path = "../utils/delta_btree_map" } +dhat = { version = "0.3", optional = true } educe = "0.6" either = "1" enum-as-inner = "0.6" @@ -32,11 +27,7 @@ fail = "0.5" foyer = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } futures-async-stream = { workspace = true } -governor = { version = "0.6", default-features = false, features = [ - "std", - "dashmap", - "jitter", -] } +governor = { workspace = true } hytra = "0.1.2" itertools = { workspace = true } jsonbb = { workspace = true } @@ -65,7 +56,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1" smallvec = "1" static_assertions = "1" -strum = "0.26" strum_macros = "0.26" thiserror = "1" thiserror-ext = { workspace = true } @@ -91,7 +81,6 @@ workspace-hack = { path = "../workspace-hack" } [dev-dependencies] assert_matches = "1" criterion = { workspace = true, features = ["async_tokio", "async"] } -dhat = "0.3" expect-test = "1" risingwave_expr_impl = { workspace = true } risingwave_hummock_sdk = { workspace = true } @@ -102,7 +91,7 @@ serde_yaml = "0.9" tracing-test = "0.2" [features] -dhat-heap = [] +dhat-heap = ["dhat"] [[bench]] name = "stream_hash_agg" diff --git a/src/stream/spill_test/Cargo.toml b/src/stream/spill_test/Cargo.toml index fe4fddfe2b1c3..b979fdfa30b80 100644 --- a/src/stream/spill_test/Cargo.toml +++ b/src/stream/spill_test/Cargo.toml @@ -8,14 +8,8 @@ license = { workspace = true } repository = { workspace = true } [dependencies] -async-trait = "0.1" -bytes = { version = "1" } -futures = { version = "0.3", default-features = false, features = ["alloc"] } -futures-async-stream = "0.2.9" risingwave_common = { workspace = true } -risingwave_hummock_sdk = { workspace = true } risingwave_hummock_test = { workspace = true, features = ["test"] } -risingwave_storage = { workspace = true, features = ["test"] } risingwave_stream = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio" } diff --git a/src/stream/src/executor/backfill/snapshot_backfill.rs b/src/stream/src/executor/backfill/snapshot_backfill.rs index 7d33ac89d403d..6851d81ce790f 100644 --- a/src/stream/src/executor/backfill/snapshot_backfill.rs +++ b/src/stream/src/executor/backfill/snapshot_backfill.rs @@ -62,6 +62,8 @@ pub struct SnapshotBackfillExecutor { actor_ctx: ActorContextRef, metrics: Arc, + + snapshot_epoch: Option, } impl SnapshotBackfillExecutor { @@ -76,6 +78,7 @@ impl SnapshotBackfillExecutor { rate_limit: Option, barrier_rx: UnboundedReceiver, metrics: Arc, + snapshot_epoch: Option, ) -> Self { assert_eq!(&upstream.info.schema, upstream_table.schema()); if let Some(rate_limit) = rate_limit { @@ -94,17 +97,37 @@ impl SnapshotBackfillExecutor { barrier_rx, actor_ctx, metrics, + snapshot_epoch, } } #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self) { debug!("snapshot backfill executor start"); - let first_barrier = expect_first_barrier(&mut self.upstream).await?; - debug!(epoch = ?first_barrier.epoch, "get first upstream barrier"); + let first_upstream_barrier = expect_first_barrier(&mut self.upstream).await?; + debug!(epoch = ?first_upstream_barrier.epoch, "get first upstream barrier"); let first_recv_barrier = receive_next_barrier(&mut self.barrier_rx).await?; debug!(epoch = ?first_recv_barrier.epoch, "get first inject barrier"); - let should_backfill = first_barrier.epoch != first_recv_barrier.epoch; + let should_backfill = if let Some(snapshot_epoch) = self.snapshot_epoch { + if first_upstream_barrier.epoch != first_recv_barrier.epoch { + assert_eq!(snapshot_epoch, first_upstream_barrier.epoch.prev); + true + } else { + false + } + } else { + // when snapshot epoch is not set, the StreamNode must be created previously and has finished the backfill + if cfg!(debug_assertions) { + panic!( + "snapshot epoch not set. first_upstream_epoch: {:?}, first_recv_epoch: {:?}", + first_upstream_barrier.epoch, first_recv_barrier.epoch + ); + } else { + warn!(first_upstream_epoch = ?first_upstream_barrier.epoch, first_recv_epoch=?first_recv_barrier.epoch, "snapshot epoch not set"); + assert_eq!(first_upstream_barrier.epoch, first_recv_barrier.epoch); + false + } + }; let (mut barrier_epoch, mut need_report_finish) = { if should_backfill { @@ -119,7 +142,7 @@ impl SnapshotBackfillExecutor { let mut upstream_buffer = UpstreamBuffer::new(&mut self.upstream, consume_upstream_row_count); - let first_barrier_epoch = first_barrier.epoch; + let first_barrier_epoch = first_upstream_barrier.epoch; // Phase 1: consume upstream snapshot { @@ -156,7 +179,7 @@ impl SnapshotBackfillExecutor { } let recv_barrier = self.barrier_rx.recv().await.expect("should exist"); - assert_eq!(first_barrier.epoch, recv_barrier.epoch); + assert_eq!(first_upstream_barrier.epoch, recv_barrier.epoch); yield Message::Barrier(recv_barrier); } @@ -239,9 +262,9 @@ impl SnapshotBackfillExecutor { table_id = self.upstream_table.table_id().table_id, "skip backfill" ); - assert_eq!(first_barrier.epoch, first_recv_barrier.epoch); + assert_eq!(first_upstream_barrier.epoch, first_recv_barrier.epoch); yield Message::Barrier(first_recv_barrier); - (first_barrier.epoch, false) + (first_upstream_barrier.epoch, false) } }; let mut upstream = self.upstream.into_executor(self.barrier_rx).execute(); diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs index f7da06608a722..ccefd82cfd2e9 100644 --- a/src/stream/src/executor/monitor/streaming_stats.rs +++ b/src/stream/src/executor/monitor/streaming_stats.rs @@ -191,7 +191,7 @@ pub struct StreamingMetrics { materialize_cache_hit_count: RelabeledGuardedIntCounterVec<3>, materialize_cache_total_count: RelabeledGuardedIntCounterVec<3>, materialize_input_row_count: RelabeledGuardedIntCounterVec<3>, - materialize_current_epoch: RelabeledGuardedIntGaugeVec<3>, + pub materialize_current_epoch: RelabeledGuardedIntGaugeVec<3>, } pub static GLOBAL_STREAMING_METRICS: OnceLock = OnceLock::new(); diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs index 1868670894182..eba1646259ea4 100644 --- a/src/stream/src/task/stream_manager.rs +++ b/src/stream/src/task/stream_manager.rs @@ -408,6 +408,7 @@ impl StreamActorManager { node.rate_limit.map(|x| x as _), barrier_rx, self.streaming_metrics.clone(), + node.snapshot_backfill_epoch, ) .boxed(); diff --git a/src/test_runner/Cargo.toml b/src/test_runner/Cargo.toml index 3b9819bd45dad..23ecd648aa57e 100644 --- a/src/test_runner/Cargo.toml +++ b/src/test_runner/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] fail = "0.5" sync-point = { path = "../utils/sync-point" } diff --git a/src/tests/compaction_test/Cargo.toml b/src/tests/compaction_test/Cargo.toml index 3bc86649ea3b3..2351e750c2a95 100644 --- a/src/tests/compaction_test/Cargo.toml +++ b/src/tests/compaction_test/Cargo.toml @@ -8,28 +8,15 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" -async-trait = "0.1" bytes = "1" clap = { workspace = true } foyer = { workspace = true } -futures = { version = "0.3", default-features = false, features = ["alloc"] } -prometheus = { version = "0.13" } -rand = { workspace = true } risingwave_common = { workspace = true } risingwave_compactor = { workspace = true } risingwave_hummock_sdk = { workspace = true } -risingwave_hummock_test = { workspace = true } -risingwave_meta = { workspace = true } risingwave_meta_node = { workspace = true } -risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_rt = { workspace = true } diff --git a/src/tests/e2e_extended_mode/Cargo.toml b/src/tests/e2e_extended_mode/Cargo.toml index 2e88f009742a7..8e4e16f2c41cd 100644 --- a/src/tests/e2e_extended_mode/Cargo.toml +++ b/src/tests/e2e_extended_mode/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = { version = "1", features = ["backtrace"] } chrono = { version = "0.4", features = ['serde'] } diff --git a/src/tests/libpq_test/Cargo.toml b/src/tests/libpq_test/Cargo.toml index 84cefac33ef6e..8e618be7928ee 100644 --- a/src/tests/libpq_test/Cargo.toml +++ b/src/tests/libpq_test/Cargo.toml @@ -7,6 +7,5 @@ version = "0.1.0" edition = "2021" [dependencies] -anyhow = "1" libpq = "3.0" clap = { workspace = true } diff --git a/src/tests/regress/Cargo.toml b/src/tests/regress/Cargo.toml index 65248877adf60..a5095d40a7f80 100644 --- a/src/tests/regress/Cargo.toml +++ b/src/tests/regress/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = { version = "1", features = ["backtrace"] } clap = { workspace = true } diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 76f2e263d7290..9fc7c348f6acd 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -5,15 +5,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [package.metadata.cargo-machete] -ignored = ["serde"] - -[package.metadata.cargo-udeps.ignore] -normal = ["serde"] +ignored = ["tikv-jemallocator"] [dependencies] anyhow = "1.0" async-trait = "0.1" -aws-sdk-s3 = { version = "0.5", package = "madsim-aws-sdk-s3" } cfg-or-panic = "0.2" clap = { workspace = true } console = "0.15" @@ -28,7 +24,6 @@ maplit = "1" paste = "1" pin-project = "1.1" pretty_assertions = "1" -prometheus = { version = "0.13" } rand = { workspace = true } rand_chacha = { version = "0.3.1" } rdkafka = { workspace = true } @@ -45,18 +40,16 @@ risingwave_hummock_sdk = { workspace = true } risingwave_meta_node = { workspace = true } risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } -risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } risingwave_sqlsmith = { workspace = true } serde = "1.0.188" serde_derive = "1.0.188" serde_json = "1.0.107" -sqllogictest = "0.23.0" +sqllogictest = "0.24.0" tempfile = "3" tikv-jemallocator = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio" } tokio-postgres = "0.7" -tokio-stream = { workspace = true } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = "*" diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml index c7cf9da9ce299..740749e58d572 100644 --- a/src/tests/sqlsmith/Cargo.toml +++ b/src/tests/sqlsmith/Cargo.toml @@ -7,12 +7,6 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" chrono = "0.4" @@ -25,7 +19,6 @@ risingwave_common = { workspace = true } risingwave_expr = { workspace = true } risingwave_expr_impl = { workspace = true } risingwave_frontend = { workspace = true } -risingwave_pb = { workspace = true } risingwave_sqlparser = { workspace = true } similar = "2.6.0" thiserror-ext = { workspace = true } diff --git a/src/tests/state_cleaning_test/Cargo.toml b/src/tests/state_cleaning_test/Cargo.toml index 6c12898343951..5a5fa90d3a85a 100644 --- a/src/tests/state_cleaning_test/Cargo.toml +++ b/src/tests/state_cleaning_test/Cargo.toml @@ -7,17 +7,10 @@ keywords = { workspace = true } license = { workspace = true } repository = { workspace = true } -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = "1" clap = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } -prometheus = { version = "0.13" } regex = "1" risingwave_rt = { workspace = true } serde = { version = "1", features = ["derive"] } diff --git a/src/utils/delta_btree_map/Cargo.toml b/src/utils/delta_btree_map/Cargo.toml index 274a028489395..879f740e89425 100644 --- a/src/utils/delta_btree_map/Cargo.toml +++ b/src/utils/delta_btree_map/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] educe = "0.6" enum-as-inner = "0.6" diff --git a/src/utils/futures_util/Cargo.toml b/src/utils/futures_util/Cargo.toml index 97bd794daaf8d..d8e763938e3e8 100644 --- a/src/utils/futures_util/Cargo.toml +++ b/src/utils/futures_util/Cargo.toml @@ -5,12 +5,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] futures = "0.3" pin-project-lite = "0.2" diff --git a/src/utils/iter_util/Cargo.toml b/src/utils/iter_util/Cargo.toml index d730c1cbc908e..2bfec2cedf147 100644 --- a/src/utils/iter_util/Cargo.toml +++ b/src/utils/iter_util/Cargo.toml @@ -5,12 +5,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] itertools = { workspace = true } diff --git a/src/utils/local_stats_alloc/Cargo.toml b/src/utils/local_stats_alloc/Cargo.toml index d80d3db38109c..b92b0b0e1359a 100644 --- a/src/utils/local_stats_alloc/Cargo.toml +++ b/src/utils/local_stats_alloc/Cargo.toml @@ -14,11 +14,5 @@ workspace-hack = { path = "../../workspace-hack" } [dev-dependencies] -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [lints] workspace = true diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml index 8f0e55ff8446b..2275ed5a20571 100644 --- a/src/utils/pgwire/Cargo.toml +++ b/src/utils/pgwire/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] anyhow = { version = "1.0", default-features = false } auto_enums = { workspace = true } diff --git a/src/utils/resource_util/Cargo.toml b/src/utils/resource_util/Cargo.toml index 9680c5054bc31..175052cdfbd16 100644 --- a/src/utils/resource_util/Cargo.toml +++ b/src/utils/resource_util/Cargo.toml @@ -5,12 +5,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] fs-err = "3" sysinfo = { version = "0.33", default-features = false, features = ["system"] } diff --git a/src/utils/runtime/Cargo.toml b/src/utils/runtime/Cargo.toml index b34cd499f9ddd..0ecadea5aeb9a 100644 --- a/src/utils/runtime/Cargo.toml +++ b/src/utils/runtime/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] await-tree = { workspace = true } console = "0.15" diff --git a/src/utils/sync-point/Cargo.toml b/src/utils/sync-point/Cargo.toml index d228bfbe79a35..27b908bc517b1 100644 --- a/src/utils/sync-point/Cargo.toml +++ b/src/utils/sync-point/Cargo.toml @@ -4,12 +4,6 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] futures-util = "0.3" spin = "0.9" diff --git a/src/utils/variables/Cargo.toml b/src/utils/variables/Cargo.toml index 7bcc1b2d963c0..c897c42986f2a 100644 --- a/src/utils/variables/Cargo.toml +++ b/src/utils/variables/Cargo.toml @@ -8,12 +8,6 @@ license = { workspace = true } repository = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[package.metadata.cargo-machete] -ignored = ["workspace-hack"] - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-hack"] - [dependencies] chrono = { version = "0.4", default-features = false, features = [ "clock",