From 735e4c92e17b2a4be782e9bd61d5ae4b668d7069 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 15 Nov 2024 16:34:37 -0500 Subject: [PATCH 01/11] fix(kubernetes_log source): Set user-agent for k8s apiserver requests. --- Cargo.lock | 246 +++++++++++------- Cargo.toml | 5 +- LICENSE-3rdparty.csv | 10 +- .../21864_kubernetes_logs_user_agent.fix.md | 3 + license-tool.toml | 3 + src/aws/auth.rs | 2 + src/kubernetes/reflector.rs | 67 +++-- src/sources/kubernetes_logs/mod.rs | 19 +- 8 files changed, 229 insertions(+), 126 deletions(-) create mode 100644 changelog.d/21864_kubernetes_logs_user_agent.fix.md diff --git a/Cargo.lock b/Cargo.lock index 225235b8e9d0c..d23e3e1a86b8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,6 +384,18 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-broadcast" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -764,12 +776,11 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.0.1" +version = "1.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-sdk-sso", "aws-sdk-ssooidc", @@ -785,11 +796,11 @@ dependencies = [ "fastrand 2.1.1", "hex", "http 0.2.9", - "hyper 0.14.28", "ring", "time", "tokio", "tracing 0.1.40", + "url", "zeroize", ] @@ -823,22 +834,26 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.0.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed7ef604a15fd0d4d9e43701295161ea6b504b63c44990ead352afea2bc15e9" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", - "aws-http", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes 1.8.0", "fastrand 2.1.1", "http 0.2.9", + "http-body 0.4.5", + "once_cell", "percent-encoding", + "pin-project-lite", "tracing 0.1.40", "uuid", ] @@ -1055,12 +1070,11 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.3.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0619ab97a5ca8982e7de073cdc66f93e5f6a1b05afc09e696bec1cb3607cd4df" +checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1071,18 +1085,18 @@ dependencies = [ "aws-types", "bytes 1.8.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.40", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.3.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04b9f5474cc0f35d829510b2ec8c21e352309b46bf9633c5a81fb9321e9b1c7" +checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1093,18 +1107,18 @@ dependencies = [ "aws-types", "bytes 1.8.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.40", ] [[package]] name = "aws-sdk-sts" -version = "1.3.1" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "798c8d82203af9e15a8b406574e0b36da91dd6db533028b74676489a1bc8bc7d" +checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1116,7 +1130,8 @@ dependencies = [ "aws-smithy-xml", "aws-types", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.40", ] @@ -1210,18 +1225,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.60.0" +version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.60.0" +version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" dependencies = [ "aws-smithy-types", "urlencoding", @@ -1299,9 +1314,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.0" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] @@ -1565,12 +1580,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" - [[package]] name = "base64" version = "0.21.7" @@ -3599,6 +3608,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" +[[package]] +name = "fluent-uri" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "flume" version = "0.10.14" @@ -4087,6 +4105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", + "allocator-api2", ] [[package]] @@ -4404,9 +4423,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes 1.8.0", "futures-util", @@ -4547,6 +4566,25 @@ dependencies = [ "tower-layer", ] +[[package]] +name = "hyper-openssl" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527d4d619ca2c2aafa31ec139a3d1d60bf557bf7578a1f20f743637eccd9ca19" +dependencies = [ + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot", + "pin-project", + "tower-layer", + "tower-service", +] + [[package]] name = "hyper-proxy" version = "0.9.1" @@ -5147,23 +5185,38 @@ dependencies = [ [[package]] name = "json-patch" -version = "1.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6" +checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" dependencies = [ + "jsonptr", "serde", "serde_json", "thiserror", - "treediff", ] [[package]] -name = "jsonpath_lib" -version = "0.3.0" +name = "jsonpath-rust" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" dependencies = [ - "log", + "lazy_static", + "once_cell", + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror", +] + +[[package]] +name = "jsonptr" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" +dependencies = [ + "fluent-uri", "serde", "serde_json", ] @@ -5201,19 +5254,15 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.18.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788" +checksum = "19501afb943ae5806548bc3ebd7f3374153ca057a38f480ef30adfde5ef09755" dependencies = [ - "base64 0.21.7", - "bytes 1.8.0", + "base64 0.22.1", "chrono", - "http 0.2.9", - "percent-encoding", "serde", "serde-value", "serde_json", - "url", ] [[package]] @@ -5267,11 +5316,11 @@ dependencies = [ [[package]] name = "kube" -version = "0.82.2" +version = "0.93.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc7d3d52dd5c871991679102e80dfb192faaaa09fecdbccdd8c55af264ce7a8f" +checksum = "0365920075af1a2d23619c1ca801c492f2400157de42627f041a061716e76416" dependencies = [ - "k8s-openapi 0.18.0", + "k8s-openapi 0.22.0", "kube-client", "kube-core", "kube-runtime", @@ -5279,27 +5328,28 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.82.2" +version = "0.93.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "544339f1665488243f79080441cacb09c997746fd763342303e66eebb9d3ba13" +checksum = "d81336eb3a5b10a40c97a5a97ad66622e92bad942ce05ee789edd730aa4f8603" dependencies = [ - "base64 0.20.0", + "base64 0.22.1", "bytes 1.8.0", "chrono", - "dirs-next", "either", "futures 0.3.31", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", - "hyper-openssl", - "hyper-timeout 0.4.1", - "jsonpath_lib", - "k8s-openapi 0.18.0", + "home", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.1", + "hyper-openssl 0.10.2", + "hyper-timeout 0.5.1", + "hyper-util", + "jsonpath-rust", + "k8s-openapi 0.22.0", "kube-core", "openssl", - "pem 1.1.1", - "pin-project", + "pem", "secrecy", "serde", "serde_json", @@ -5308,22 +5358,21 @@ dependencies = [ "tokio", "tokio-util", "tower", - "tower-http", + "tower-http 0.5.2", "tracing 0.1.40", ] [[package]] name = "kube-core" -version = "0.82.2" +version = "0.93.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25983d07f414dfffba08c5951fe110f649113416b1d8e22f7c89c750eb2555a7" +checksum = "cce373a74d787d439063cdefab0f3672860bd7bac01a38e39019177e764a0fe6" dependencies = [ "chrono", "form_urlencoded", - "http 0.2.9", + "http 1.1.0", "json-patch", - "k8s-openapi 0.18.0", - "once_cell", + "k8s-openapi 0.22.0", "serde", "serde_json", "thiserror", @@ -5331,23 +5380,26 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.82.2" +version = "0.93.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125331201e3073707ac79c294c89021faa76c84da3a566a3749a2a93d295c98a" +checksum = "3b84733c0fed6085c9210b43ffb96248676c1e800d0ba38d15043275a792ffa4" dependencies = [ "ahash 0.8.11", + "async-broadcast", + "async-stream", "async-trait", "backoff", "derivative", "futures 0.3.31", + "hashbrown 0.14.5", "json-patch", - "k8s-openapi 0.18.0", + "jsonptr", + "k8s-openapi 0.22.0", "kube-client", "parking_lot", "pin-project", "serde", "serde_json", - "smallvec", "thiserror", "tokio", "tokio-util", @@ -6795,15 +6847,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518" -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "3.0.2" @@ -7600,7 +7643,7 @@ dependencies = [ "nom", "oauth2", "openidconnect", - "pem 3.0.2", + "pem", "prost 0.11.9", "prost-build 0.11.9", "prost-derive 0.11.9", @@ -10082,7 +10125,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", - "base64 0.21.7", "bitflags 2.4.1", "bytes 1.8.0", "futures-core", @@ -10090,7 +10132,6 @@ dependencies = [ "http 0.2.9", "http-body 0.4.5", "http-range-header", - "mime", "pin-project-lite", "tokio", "tokio-util", @@ -10099,6 +10140,25 @@ dependencies = [ "tracing 0.1.40", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "base64 0.21.7", + "bitflags 2.4.1", + "bytes 1.8.0", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing 0.1.40", +] + [[package]] name = "tower-layer" version = "0.3.2" @@ -10272,15 +10332,6 @@ dependencies = [ "tracing-futures 0.3.0", ] -[[package]] -name = "treediff" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" -dependencies = [ - "serde_json", -] - [[package]] name = "triomphe" version = "0.1.12" @@ -10772,17 +10823,18 @@ dependencies = [ "hickory-proto", "hostname 0.4.0", "http 0.2.9", + "http 1.1.0", "http-body 0.4.5", "http-serde", "hyper 0.14.28", - "hyper-openssl", + "hyper-openssl 0.9.2", "hyper-proxy", "indexmap 2.6.0", "indoc", "inventory", "ipnet", "itertools 0.13.0", - "k8s-openapi 0.18.0", + "k8s-openapi 0.22.0", "kube", "lapin", "libc", @@ -10863,7 +10915,7 @@ dependencies = [ "tonic 0.11.0", "tonic-build 0.11.0", "tower", - "tower-http", + "tower-http 0.4.4", "tower-test", "tracing 0.1.40", "tracing-core 0.1.32", diff --git a/Cargo.toml b/Cargo.toml index d69052f5fc645..a7d30948d16e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,6 +307,7 @@ hashbrown = { version = "0.14.5", default-features = false, optional = true, fea headers = { version = "0.3.9", default-features = false } hostname = { version = "0.4.0", default-features = false } http = { version = "0.2.9", default-features = false } +http-1 = { package = "http", version = "1.0", default-features = false } http-serde = "1.1.3" http-body = { version = "0.4.5", default-features = false } hyper = { version = "0.14.28", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } @@ -317,8 +318,8 @@ indoc = { version = "2.0.5", default-features = false } inventory = { version = "0.3.15", default-features = false } ipnet = { version = "2", default-features = false, optional = true, features = ["serde", "std"] } itertools = { version = "0.13.0", default-features = false, optional = false, features = ["use_alloc"] } -k8s-openapi = { version = "0.18.0", default-features = false, features = ["api", "v1_26"], optional = true } -kube = { version = "0.82.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } +k8s-openapi = { version = "0.22.0", default-features = false, features = ["v1_26"], optional = true } +kube = { version = "0.93.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } listenfd = { version = "1.0.1", default-features = false, optional = true } logfmt = { version = "0.0.2", default-features = false, optional = true } lru = { version = "0.12.5", default-features = false, optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index a44210b1c043c..27fb14c215f08 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -28,6 +28,7 @@ arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Va arr_macro,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss ascii,https://github.com/tomprogrammer/rust-ascii,Apache-2.0 OR MIT,"Thomas Bahn , Torbjørn Birch Moltu , Simon Sapin " +async-broadcast,https://github.com/smol-rs/async-broadcast,MIT OR Apache-2.0,"Stjepan Glavina , Yoshua Wuyts , Zeeshan Ali Khan " async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina async-compression,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " async-executor,https://github.com/smol-rs/async-executor,Apache-2.0 OR MIT,Stjepan Glavina @@ -224,6 +225,7 @@ finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_ flagset,https://github.com/enarx/flagset,Apache-2.0,Nathaniel McCallum flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float_eq,https://github.com/jtempest/float_eq-rs,MIT OR Apache-2.0,jtempest +fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto fnv,https://github.com/servo/rust-fnv,Apache-2.0 OR MIT,Alex Crichton foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters @@ -325,15 +327,14 @@ jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase js-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers json-patch,https://github.com/idubrov/json-patch,MIT OR Apache-2.0,Ivan Dubrov -jsonpath_lib,https://github.com/freestrings/jsonpath,MIT,Changseok Han -k8s-openapi,https://github.com/Arnavion/k8s-openapi,Apache-2.0,Arnavion +jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev +jsonptr,https://github.com/chanced/jsonptr,MIT OR Apache-2.0,chance dinkins +k8s-openapi,https://github.com/Arnavion/k8s-openapi,Apache-2.0,Arnav Singh keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MIT,RustCrypto Developers kqueue,https://gitlab.com/rust-kqueue/rust-kqueue,MIT,William Orr kqueue-sys,https://gitlab.com/rust-kqueue/rust-kqueue-sys,MIT,"William Orr , Daniel (dmilith) Dettlaff " krb5-src,https://github.com/MaterializeInc/rust-krb5-src,Apache-2.0,"Materialize, Inc." kube,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " -kube-core,https://github.com/kube-rs/kube,Apache-2.0,"clux , kazk " -kube-runtime,https://github.com/kube-rs/kube,Apache-2.0,"Natalie Klestrup Röijezon , clux " lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie , Marc-Antoine Perennou " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel @@ -632,7 +633,6 @@ tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " tracing-tower,https://github.com/tokio-rs/tracing,MIT,Eliza Weisman -treediff,https://github.com/Byron/treediff-rs,MIT OR Apache-2.0,Sebastian Thiel triomphe,https://github.com/Manishearth/triomphe,MIT OR Apache-2.0,"Manish Goregaokar , The Servo Project Developers" trust-dns-proto,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry trust-dns-resolver,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry diff --git a/changelog.d/21864_kubernetes_logs_user_agent.fix.md b/changelog.d/21864_kubernetes_logs_user_agent.fix.md new file mode 100644 index 0000000000000..81a8f4b78af5b --- /dev/null +++ b/changelog.d/21864_kubernetes_logs_user_agent.fix.md @@ -0,0 +1,3 @@ +The `kubernetes_log` source now sets a user-agent when querying k8s apiserver. + +authors: ganelo diff --git a/license-tool.toml b/license-tool.toml index 6a91a44f04184..5696308a86e0a 100644 --- a/license-tool.toml +++ b/license-tool.toml @@ -4,6 +4,9 @@ "openssl-macros" = { origin = "https://github.com/sfackler/rust-openssl" } "serde_nanos" = { origin = "https://github.com/caspervonb/serde_nanos" } +# rust-license-tool can't find the license for jsonpath-rust 0.5.1 +"jsonpath-rust" = { license = "MIT", origin = "https://github.com/besok/jsonpath-rust" } + # `ring` has a custom license that is mostly "ISC-style" but parts of it also fall under OpenSSL licensing. "ring-0.16.20" = { license = "ISC AND Custom" } "ring-0.17.5" = { license = "ISC AND Custom" } diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 7671c039af0e8..b220e84f436c0 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -1,6 +1,7 @@ //! Authentication settings for AWS components. use std::time::Duration; +#[allow(deprecated)] use aws_config::{ default_provider::credentials::DefaultCredentialsChain, identity::IdentityCache, @@ -275,6 +276,7 @@ impl AwsAuthentication { } => { let connector = super::connector(proxy, tls_options)?; + #[allow(deprecated)] // The SDK uses the default profile out of the box, but doesn't provide an optional // type in the builder. We can just hardcode it so that everything works. let profile_files = ProfileFiles::builder() diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 1135fd4c42c53..14460c73981a6 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -1,6 +1,6 @@ //! Intercept [`watcher::Event`]'s. -use std::{hash::Hash, time::Duration}; +use std::{hash::Hash, sync::Arc, time::Duration}; use futures::StreamExt; use futures_util::Stream; @@ -26,31 +26,64 @@ pub async fn custom_reflector( { pin!(stream); let mut delay_queue = DelayQueue::default(); + let mut init_buffer_meta = Vec::new(); loop { tokio::select! { result = stream.next() => { match result { Some(Ok(event)) => { match event { - // Immediately reconcile `Applied` event - watcher::Event::Applied(ref obj) => { - trace!(message = "Processing Applied event.", ?event); + // Immediately reconcile `Apply` event + watcher::Event::Apply(ref obj) => { + trace!(message = "Processing Apply event.", event_type = std::any::type_name::(), event = ?event); store.apply_watcher_event(&event); let meta_descr = MetaDescribe::from_meta(obj.meta()); meta_cache.store(meta_descr); } - // Delay reconciling any `Deleted` events - watcher::Event::Deleted(ref obj) => { + // Delay reconciling any `Delete` events + watcher::Event::Delete(ref obj) => { + trace!(message = "Delaying processing Delete event.", event_type = std::any::type_name::(), event = ?event); delay_queue.insert(event.to_owned(), delay_deletion); let meta_descr = MetaDescribe::from_meta(obj.meta()); meta_cache.delete(&meta_descr); } - // Clear all delayed events on `Restarted` events - watcher::Event::Restarted(_) => { - trace!(message = "Processing Restarted event.", ?event); + // Clear all delayed events on `Init` event + watcher::Event::Init => { + trace!(message = "Processing Init event.", event_type = std::any::type_name::(), event = ?event); delay_queue.clear(); store.apply_watcher_event(&event); meta_cache.clear(); + init_buffer_meta.clear(); + } + // Immediately reconcile `InitApply` event (but buffer the obj ref so we can handle implied deletions on InitDone) + watcher::Event::InitApply(ref obj) => { + trace!(message = "Processing InitApply event.", event_type = std::any::type_name::(), event = ?event); + store.apply_watcher_event(&event); + let meta_descr = MetaDescribe::from_meta(obj.meta()); + meta_cache.store(meta_descr.clone()); + init_buffer_meta.push(meta_descr.clone()); + } + // Reconcile `InitApply` events and implied deletions + watcher::Event::InitDone => { + trace!(message = "Processing InitDone event.", event_type = std::any::type_name::(), event = ?event); + store.apply_watcher_event(&event); + + + store.as_reader().state().into_iter() + // delay deleting objs that were added before but not during InitApply + .for_each(|obj| { + if let Some(inner) = Arc::into_inner(obj) { + let meta_descr = MetaDescribe::from_meta(inner.meta()); + if !init_buffer_meta.contains(&meta_descr) { + let implied_deletion_event = watcher::Event::Delete(inner); + trace!(message = "Delaying processing implied deletion.", event_type = std::any::type_name::(), event = ?implied_deletion_event); + delay_queue.insert(implied_deletion_event, delay_deletion); + meta_cache.delete(&meta_descr); + } + } + }); + + init_buffer_meta.clear(); } } }, @@ -69,10 +102,10 @@ pub async fn custom_reflector( Some(event) => { let event = event.into_inner(); match event { - watcher::Event::Deleted(ref obj) => { + watcher::Event::Delete(ref obj) => { let meta_desc = MetaDescribe::from_meta(obj.meta()); if !meta_cache.contains(&meta_desc) { - trace!(message = "Processing Deleted event.", ?event); + trace!(message = "Processing Delete event.", event_type = std::any::type_name::(), event = ?event); store.apply_watcher_event(&event); } }, @@ -118,7 +151,7 @@ mod tests { ..ConfigMap::default() }; let (mut tx, rx) = mpsc::channel::<_>(5); - tx.send(Ok(watcher::Event::Applied(cm.clone()))) + tx.send(Ok(watcher::Event::Apply(cm.clone()))) .await .unwrap(); let meta_cache = MetaCache::new(); @@ -144,10 +177,10 @@ mod tests { ..ConfigMap::default() }; let (mut tx, rx) = mpsc::channel::<_>(5); - tx.send(Ok(watcher::Event::Applied(cm.clone()))) + tx.send(Ok(watcher::Event::Apply(cm.clone()))) .await .unwrap(); - tx.send(Ok(watcher::Event::Deleted(cm.clone()))) + tx.send(Ok(watcher::Event::Delete(cm.clone()))) .await .unwrap(); let meta_cache = MetaCache::new(); @@ -178,13 +211,13 @@ mod tests { ..ConfigMap::default() }; let (mut tx, rx) = mpsc::channel::<_>(5); - tx.send(Ok(watcher::Event::Applied(cm.clone()))) + tx.send(Ok(watcher::Event::Apply(cm.clone()))) .await .unwrap(); - tx.send(Ok(watcher::Event::Deleted(cm.clone()))) + tx.send(Ok(watcher::Event::Delete(cm.clone()))) .await .unwrap(); - tx.send(Ok(watcher::Event::Applied(cm.clone()))) + tx.send(Ok(watcher::Event::Apply(cm.clone()))) .await .unwrap(); let meta_cache = MetaCache::new(); diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 752111e73dac5..ef0f7bd947023 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -10,6 +10,7 @@ use bytes::Bytes; use chrono::Utc; use futures::{future::FutureExt, stream::StreamExt}; use futures_util::Stream; +use http_1::{HeaderName, HeaderValue}; use k8s_openapi::api::core::v1::{Namespace, Node, Pod}; use k8s_paths_provider::K8sPathsProvider; use kube::{ @@ -34,7 +35,10 @@ use vector_lib::{ }; use vrl::value::{kind::Collection, Kind}; -use crate::sources::kubernetes_logs::partial_events_merger::merge_partial_events; +use crate::{ + built_info::{PKG_NAME, PKG_VERSION}, + sources::kubernetes_logs::partial_events_merger::merge_partial_events, +}; use crate::{ config::{ log_schema, ComponentKey, DataType, GenerateConfig, GlobalOptions, SourceConfig, @@ -586,7 +590,7 @@ impl Source { // If the user passed a custom Kubeconfig use it, otherwise // we attempt to load the local kubeconfig, followed by the // in-cluster environment variables - let client_config = match &config.kube_config_file { + let mut client_config = match &config.kube_config_file { Some(kc) => { ClientConfig::from_custom_kubeconfig( config::Kubeconfig::read_from(kc)?, @@ -596,6 +600,11 @@ impl Source { } None => ClientConfig::infer().await?, }; + if let Ok(user_agent) = HeaderValue::from_str(&format!("{}/{}", PKG_NAME, PKG_VERSION)) { + client_config + .headers + .push((HeaderName::from_static("user-agent"), user_agent)); + } let client = Client::try_from(client_config)?; let data_dir = globals.resolve_and_make_data_subdir(config.data_dir.as_ref(), key.id())?; @@ -695,7 +704,7 @@ impl Source { ..Default::default() }, ) - .backoff(watcher::default_backoff()); + .backoff(watcher::DefaultBackoff::default()); let pod_store_w = reflector::store::Writer::default(); let pod_state = pod_store_w.as_reader(); let pod_cacher = MetaCache::new(); @@ -718,7 +727,7 @@ impl Source { ..Default::default() }, ) - .backoff(watcher::default_backoff()); + .backoff(watcher::DefaultBackoff::default()); let ns_store_w = reflector::store::Writer::default(); let ns_state = ns_store_w.as_reader(); let ns_cacher = MetaCache::new(); @@ -741,7 +750,7 @@ impl Source { ..Default::default() }, ) - .backoff(watcher::default_backoff()); + .backoff(watcher::DefaultBackoff::default()); let node_store_w = reflector::store::Writer::default(); let node_state = node_store_w.as_reader(); let node_cacher = MetaCache::new(); From c3cc34312d614ac3fd8670264107b8520c536092 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 6 Dec 2024 13:42:33 -0500 Subject: [PATCH 02/11] Update changelog.d/21864_kubernetes_logs_user_agent.fix.md Co-authored-by: Pavlos Rontidis --- changelog.d/21864_kubernetes_logs_user_agent.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/21864_kubernetes_logs_user_agent.fix.md b/changelog.d/21864_kubernetes_logs_user_agent.fix.md index 81a8f4b78af5b..e438f437f304a 100644 --- a/changelog.d/21864_kubernetes_logs_user_agent.fix.md +++ b/changelog.d/21864_kubernetes_logs_user_agent.fix.md @@ -1,3 +1,3 @@ -The `kubernetes_log` source now sets a user-agent when querying k8s apiserver. +The `kubernetes_log` source now sets a `user-agent` header when querying [k8s apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). authors: ganelo From 4ac20b3146563863500f48b74fa98ae6f87dff52 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 6 Dec 2024 14:01:44 -0500 Subject: [PATCH 03/11] Switch away from deprecated structs --- Cargo.lock | 1 + Cargo.toml | 2 ++ src/aws/auth.rs | 12 ++++-------- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d23e3e1a86b8e..d6c7597b47e8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10761,6 +10761,7 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", + "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", "aws-sdk-elasticsearch", diff --git a/Cargo.toml b/Cargo.toml index a7d30948d16e6..b5239c6b9a4e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -215,6 +215,7 @@ aws-types = { version = "1.3.3", default-features = false, optional = true } aws-sigv4 = { version = "1.2.5", default-features = false, features = ["sign-http"], optional = true } aws-config = { version = "1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } +aws-runtime = { version = "1.4.3", optional = true } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } aws-smithy-types = { version = "1.2.9", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } @@ -487,6 +488,7 @@ api-client = [ aws-core = [ "aws-config", + "aws-runtime", "dep:aws-credential-types", "dep:aws-sigv4", "dep:aws-types", diff --git a/src/aws/auth.rs b/src/aws/auth.rs index b220e84f436c0..720af9ee0e0f6 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -1,19 +1,16 @@ //! Authentication settings for AWS components. use std::time::Duration; -#[allow(deprecated)] use aws_config::{ default_provider::credentials::DefaultCredentialsChain, identity::IdentityCache, imds, - profile::{ - profile_file::{ProfileFileKind, ProfileFiles}, - ProfileFileCredentialsProvider, - }, + profile::ProfileFileCredentialsProvider, provider_config::ProviderConfig, sts::AssumeRoleProviderBuilder, }; use aws_credential_types::{provider::SharedCredentialsProvider, Credentials}; +use aws_runtime::env_config::file::{EnvConfigFileKind, EnvConfigFiles}; use aws_smithy_async::time::SystemTimeSource; use aws_smithy_runtime_api::client::identity::SharedIdentityCache; use aws_types::{region::Region, SdkConfig}; @@ -276,11 +273,10 @@ impl AwsAuthentication { } => { let connector = super::connector(proxy, tls_options)?; - #[allow(deprecated)] // The SDK uses the default profile out of the box, but doesn't provide an optional // type in the builder. We can just hardcode it so that everything works. - let profile_files = ProfileFiles::builder() - .with_file(ProfileFileKind::Credentials, credentials_file) + let profile_files = EnvConfigFiles::builder() + .with_file(EnvConfigFileKind::Credentials, credentials_file) .build(); let provider_config = ProviderConfig::empty().with_http_client(connector); From 3d63059ab2761410c169f2e7e80c9a57715030fb Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 6 Dec 2024 16:28:25 -0500 Subject: [PATCH 04/11] Update 21864_kubernetes_logs_user_agent.fix.md --- changelog.d/21864_kubernetes_logs_user_agent.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/21864_kubernetes_logs_user_agent.fix.md b/changelog.d/21864_kubernetes_logs_user_agent.fix.md index e438f437f304a..069cc0d833108 100644 --- a/changelog.d/21864_kubernetes_logs_user_agent.fix.md +++ b/changelog.d/21864_kubernetes_logs_user_agent.fix.md @@ -1,3 +1,3 @@ -The `kubernetes_log` source now sets a `user-agent` header when querying [k8s apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). +The `kubernetes_logs` source now sets a `user-agent` header when querying [k8s apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). authors: ganelo From 2804e985879e822a5059b334f05a587bce427369 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Tue, 10 Dec 2024 13:48:13 -0500 Subject: [PATCH 05/11] Try manually updating version via crate update --- Cargo.lock | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90655b2dc5fb5..d08f8c0c71d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -776,11 +776,12 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.5.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" dependencies = [ "aws-credential-types", + "aws-http", "aws-runtime", "aws-sdk-sso", "aws-sdk-ssooidc", @@ -796,11 +797,11 @@ dependencies = [ "fastrand 2.1.1", "hex", "http 0.2.9", + "hyper 0.14.28", "ring", "time", "tokio", "tracing 0.1.41", - "url", "zeroize", ] @@ -4509,7 +4510,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.8", "tokio", "tower-service", "tracing 0.1.41", From c6f082b8be220daa86145b76712881ab4f6b55b1 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Tue, 10 Dec 2024 13:50:49 -0500 Subject: [PATCH 06/11] Specify using tilde in Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b7ffb9e1d794e..df2fa6f583a96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -225,7 +225,7 @@ aws-sdk-secretsmanager = { version = "1.3.0", default-features = false, features aws-sdk-sts = { version = "1.3.1", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-types = { version = "1.3.3", default-features = false, optional = true } aws-sigv4 = { version = "1.2.5", default-features = false, features = ["sign-http"], optional = true } -aws-config = { version = "1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } +aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } aws-runtime = { version = "1.4.3", optional = true } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } From d504badc57c3f2d43fba6f15edfd20c6c3b0982d Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 13 Dec 2024 12:02:06 -0500 Subject: [PATCH 07/11] Rip out unused aws-runtime dep --- Cargo.lock | 1 - Cargo.toml | 2 -- src/aws/auth.rs | 6 +++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d08f8c0c71d1c..a4b2a7fae82ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10836,7 +10836,6 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", - "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", "aws-sdk-elasticsearch", diff --git a/Cargo.toml b/Cargo.toml index df2fa6f583a96..fea30ad9ed15c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -227,7 +227,6 @@ aws-types = { version = "1.3.3", default-features = false, optional = true } aws-sigv4 = { version = "1.2.5", default-features = false, features = ["sign-http"], optional = true } aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } -aws-runtime = { version = "1.4.3", optional = true } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } aws-smithy-types = { version = "1.2.9", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } @@ -500,7 +499,6 @@ api-client = [ aws-core = [ "aws-config", - "aws-runtime", "dep:aws-credential-types", "dep:aws-sigv4", "dep:aws-types", diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 720af9ee0e0f6..f342176a4815f 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -1,6 +1,7 @@ //! Authentication settings for AWS components. use std::time::Duration; +use aws_config::profile::profile_file::{ProfileFileKind, ProfileFiles}; use aws_config::{ default_provider::credentials::DefaultCredentialsChain, identity::IdentityCache, @@ -10,7 +11,6 @@ use aws_config::{ sts::AssumeRoleProviderBuilder, }; use aws_credential_types::{provider::SharedCredentialsProvider, Credentials}; -use aws_runtime::env_config::file::{EnvConfigFileKind, EnvConfigFiles}; use aws_smithy_async::time::SystemTimeSource; use aws_smithy_runtime_api::client::identity::SharedIdentityCache; use aws_types::{region::Region, SdkConfig}; @@ -275,8 +275,8 @@ impl AwsAuthentication { // The SDK uses the default profile out of the box, but doesn't provide an optional // type in the builder. We can just hardcode it so that everything works. - let profile_files = EnvConfigFiles::builder() - .with_file(EnvConfigFileKind::Credentials, credentials_file) + let profile_files = ProfileFiles::builder() + .with_file(ProfileFileKind::Credentials, credentials_file) .build(); let provider_config = ProviderConfig::empty().with_http_client(connector); From de8e5fc3019686d0f84a3fd826689936df5a0dd1 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 13 Dec 2024 12:08:04 -0500 Subject: [PATCH 08/11] Try to limit dep bumps --- Cargo.lock | 41 +++++++++++++++++++---------------------- Cargo.toml | 17 +++++++++-------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4b2a7fae82ce..9fdb7d7966c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,26 +835,22 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "ce0953f7fc1c4428511345e28ea3e98c8b59c9e91eafae30bf76d71d70642693" dependencies = [ "aws-credential-types", + "aws-http", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", "fastrand 2.1.1", "http 0.2.9", - "http-body 0.4.5", - "once_cell", "percent-encoding", - "pin-project-lite", "tracing 0.1.41", "uuid", ] @@ -1071,11 +1067,12 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.49.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" +checksum = "86575c7604dcdb583aba3390200e5333d8e4fe597bad54f57b190aaf4fac9771" dependencies = [ "aws-credential-types", + "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1086,18 +1083,18 @@ dependencies = [ "aws-types", "bytes 1.9.0", "http 0.2.9", - "once_cell", - "regex-lite", + "regex", "tracing 0.1.41", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.50.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" +checksum = "bef0d7c1d0730adb5e85407174483a579e39576e0f4350ecd0fac69ec1217b1b" dependencies = [ "aws-credential-types", + "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1108,18 +1105,18 @@ dependencies = [ "aws-types", "bytes 1.9.0", "http 0.2.9", - "once_cell", - "regex-lite", + "regex", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sts" -version = "1.50.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +checksum = "f45778089751d5aa8645a02dd60865fa0eea39f00be5db2c7779bc50b83db19a" dependencies = [ "aws-credential-types", + "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1131,8 +1128,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "http 0.2.9", - "once_cell", - "regex-lite", + "regex", "tracing 0.1.41", ] @@ -7502,7 +7498,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes 1.9.0", "heck 0.5.0", - "itertools 0.11.0", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -7535,7 +7531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2 1.0.92", "quote 1.0.37", "syn 2.0.90", @@ -7548,7 +7544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.13.0", "proc-macro2 1.0.92", "quote 1.0.37", "syn 2.0.90", @@ -10836,6 +10832,7 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", + "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", "aws-sdk-elasticsearch", diff --git a/Cargo.toml b/Cargo.toml index fea30ad9ed15c..354e91703e5ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -211,14 +211,14 @@ metrics-tracing-context.workspace = true # AWS - Official SDK aws-sdk-s3 = { version = "1.4.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sns = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-secretsmanager = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sqs = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sns = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatch = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatchlogs = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-elasticsearch = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-firehose = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kinesis = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-secretsmanager = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } # The sts crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication # is configured. @@ -227,6 +227,7 @@ aws-types = { version = "1.3.3", default-features = false, optional = true } aws-sigv4 = { version = "1.2.5", default-features = false, features = ["sign-http"], optional = true } aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } +aws-runtime = { version = "~1.0.1" } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } aws-smithy-types = { version = "1.2.9", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } From fab73f1acf685d35df2c2d2fdc4ba8a2c0aa843d Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Fri, 13 Dec 2024 12:09:57 -0500 Subject: [PATCH 09/11] Revert superfluous import change --- src/aws/auth.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aws/auth.rs b/src/aws/auth.rs index f342176a4815f..7671c039af0e8 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -1,12 +1,14 @@ //! Authentication settings for AWS components. use std::time::Duration; -use aws_config::profile::profile_file::{ProfileFileKind, ProfileFiles}; use aws_config::{ default_provider::credentials::DefaultCredentialsChain, identity::IdentityCache, imds, - profile::ProfileFileCredentialsProvider, + profile::{ + profile_file::{ProfileFileKind, ProfileFiles}, + ProfileFileCredentialsProvider, + }, provider_config::ProviderConfig, sts::AssumeRoleProviderBuilder, }; From cc4196f9860710b954f1bd939134d33a852abb77 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Mon, 16 Dec 2024 17:44:24 -0500 Subject: [PATCH 10/11] Remove superfluous pins --- Cargo.lock | 1 - Cargo.toml | 17 ++++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fdb7d7966c20..d067e40146622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10832,7 +10832,6 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", - "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", "aws-sdk-elasticsearch", diff --git a/Cargo.toml b/Cargo.toml index 354e91703e5ee..fea30ad9ed15c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -211,14 +211,14 @@ metrics-tracing-context.workspace = true # AWS - Official SDK aws-sdk-s3 = { version = "1.4.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sqs = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sns = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatch = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatchlogs = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-elasticsearch = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-firehose = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-kinesis = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-secretsmanager = { version = "~1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sns = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-secretsmanager = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } # The sts crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication # is configured. @@ -227,7 +227,6 @@ aws-types = { version = "1.3.3", default-features = false, optional = true } aws-sigv4 = { version = "1.2.5", default-features = false, features = ["sign-http"], optional = true } aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } -aws-runtime = { version = "~1.0.1" } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } aws-smithy-types = { version = "1.2.9", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } From addc91fc3a0417c4bb27618ce9641aaa84333461 Mon Sep 17 00:00:00 2001 From: Orri Ganel Date: Tue, 17 Dec 2024 13:37:44 -0500 Subject: [PATCH 11/11] Try to fix compilation error in merge jobs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fea30ad9ed15c..16b98ec10b1f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -319,7 +319,7 @@ hashbrown = { version = "0.14.5", default-features = false, optional = true, fea headers = { version = "0.3.9", default-features = false } hostname = { version = "0.4.0", default-features = false } http = { version = "0.2.9", default-features = false } -http-1 = { package = "http", version = "1.0", default-features = false } +http-1 = { package = "http", version = "1.0", default-features = false, features = ["std"] } http-serde = "1.1.3" http-body = { version = "0.4.5", default-features = false } hyper = { version = "0.14.28", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] }