diff --git a/Cargo.lock b/Cargo.lock index 0625b4de69e733..019863e468eeda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2799,6 +2799,37 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "derive_builder" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +dependencies = [ + "darling 0.20.8", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 2.0.66", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +dependencies = [ + "derive_builder_core", + "syn 2.0.66", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -3747,23 +3778,24 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.4.1#4306ab645ee55b3f7f2ad3fb7acc5820f967c1aa" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.7.0#4bc0d17577dbea47396a064c1ccf229a4c9539fa" dependencies = [ "prost 0.12.6", "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.10.2", - "tonic-build 0.10.2", + "tonic 0.11.0", + "tonic-build 0.11.0", ] [[package]] -name = "greptimedb-client" +name = "greptimedb-ingester" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust.git?rev=d21dbcff680139ed2065b62100bac3123da7c789#d21dbcff680139ed2065b62100bac3123da7c789" +source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d#2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d" dependencies = [ "dashmap 5.5.3", + "derive_builder", "enum_dispatch", "futures 0.3.30", "futures-util", @@ -3774,7 +3806,7 @@ dependencies = [ "snafu 0.7.5", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.11.0", "tonic-build 0.9.2", "tower", ] @@ -9576,6 +9608,7 @@ dependencies = [ "axum", "base64 0.21.7", "bytes 1.6.0", + "flate2", "h2 0.3.26", "http 0.2.9", "http-body 0.4.5", @@ -9584,12 +9617,17 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.12.6", + "rustls-native-certs 0.7.0", + "rustls-pemfile 2.1.0", + "rustls-pki-types", "tokio", + "tokio-rustls 0.25.0", "tokio-stream", "tower", "tower-layer", "tower-service", "tracing 0.1.40", + "zstd 0.12.4", ] [[package]] @@ -9618,6 +9656,19 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "tonic-build" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +dependencies = [ + "prettyplease 0.2.15", + "proc-macro2 1.0.86", + "prost-build 0.12.6", + "quote 1.0.36", + "syn 2.0.66", +] + [[package]] name = "tower" version = "0.4.13" @@ -10312,7 +10363,7 @@ dependencies = [ "glob", "goauth", "governor", - "greptimedb-client", + "greptimedb-ingester", "grok", "h2 0.4.5", "hash_hasher", diff --git a/Cargo.toml b/Cargo.toml index 7d4979cf02ebd8..73d5e48a3dd0e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,7 +252,7 @@ ratatui = { version = "0.26.3", optional = true, default-features = false, featu hex = { version = "0.4.3", default-features = false, optional = true } # GreptimeDB -greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust.git", rev = "d21dbcff680139ed2065b62100bac3123da7c789", optional = true } +greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d", optional = true } # External libs arc-swap = { version = "1.7", default-features = false, optional = true } @@ -674,6 +674,7 @@ sinks-logs = [ "sinks-elasticsearch", "sinks-file", "sinks-gcp", + "sinks-greptimedb_logs", "sinks-honeycomb", "sinks-http", "sinks-humio", @@ -737,7 +738,8 @@ sinks-elasticsearch = ["transforms-metric_to_log"] sinks-file = ["dep:async-compression"] sinks-gcp = ["sinks-gcp-chronicle", "dep:base64", "gcp"] sinks-gcp-chronicle = ["gcp"] -sinks-greptimedb = ["dep:greptimedb-client"] +sinks-greptimedb = ["dep:greptimedb-ingester"] +sinks-greptimedb_logs = [] sinks-honeycomb = [] sinks-http = [] sinks-humio = ["sinks-splunk_hec", "transforms-metric_to_log"] @@ -787,6 +789,7 @@ all-integration-tests = [ "gcp-integration-tests", "gcp-pubsub-integration-tests", "greptimedb-integration-tests", + "greptimedb-logs-integration-tests", "http-client-integration-tests", "humio-integration-tests", "influxdb-integration-tests", @@ -851,6 +854,7 @@ gcp-cloud-storage-integration-tests = ["sinks-gcp"] gcp-integration-tests = ["sinks-gcp"] gcp-pubsub-integration-tests = ["sinks-gcp", "sources-gcp_pubsub"] greptimedb-integration-tests = ["sinks-greptimedb"] +greptimedb-logs-integration-tests = ["sinks-greptimedb_logs"] humio-integration-tests = ["sinks-humio"] http-client-integration-tests = ["sources-http_client"] influxdb-integration-tests = ["sinks-influxdb"] diff --git a/src/sinks/greptimedb/logs/config.rs b/src/sinks/greptimedb/logs/config.rs new file mode 100644 index 00000000000000..f8a7c9d48ea2ed --- /dev/null +++ b/src/sinks/greptimedb/logs/config.rs @@ -0,0 +1,171 @@ +use std::collections::HashMap; + +use vector_lib::configurable::configurable_component; +use vector_lib::sensitive_string::SensitiveString; + +use crate::http::{Auth, HttpClient}; +use crate::sinks::greptimedb::logs::http_request_builder::{ + http_healthcheck, GreptimeDBHttpRetryLogic, GreptimeDBLogsHttpRequestBuilder, PartitionKey, +}; +use crate::sinks::greptimedb::logs::sink::GreptimeDBLogsHttpSink; +use crate::sinks::util::http::HttpService; +use crate::sinks::{ + greptimedb::{default_dbname_template, GreptimeDBDefaultBatchSettings}, + prelude::*, +}; +use vector_lib::codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoderConfig}; + +fn extra_params_examples() -> HashMap { + HashMap::<_, _>::from_iter([("source".to_owned(), "vector".to_owned())]) +} + +/// Configuration for the `greptimedb_logs` sink. +#[configurable_component(sink("greptimedb_logs", "Ingest logs data into GreptimeDB."))] +#[derive(Clone, Debug, Default, Derivative)] +#[serde(deny_unknown_fields)] +pub struct GreptimeDBLogsConfig { + /// The endpoint of the GreptimeDB server. + #[serde(alias = "host")] + #[configurable(metadata(docs::examples = "http://localhost:4000"))] + pub endpoint: String, + + /// The table that data is inserted into. + #[configurable(metadata(docs::examples = "mytable"))] + pub table: Template, + + /// The GreptimeDB [database][database] name to connect. + /// + /// Default to `public`, the default database of GreptimeDB. + /// + /// Database can be created via `create database` statement on + /// GreptimeDB. If you are using GreptimeCloud, use `dbname` from the + /// connection information of your instance. + /// + /// [database]: https://docs.greptime.com/user-guide/concepts/key-concepts#database + #[configurable(metadata(docs::examples = "public"))] + #[derivative(Default(value = "default_dbname_template()"))] + #[serde(default = "default_dbname_template")] + pub dbname: Template, + + /// pipeline name to be used for the logs + #[configurable(metadata(docs::examples = "pipeline_name"))] + pub pipeline_name: Template, + + /// pipeline version to be used for the logs + #[configurable(metadata(docs::examples = "2024-06-07 06:46:23.858293"))] + pub pipeline_version: Option