From 55b563d64e9bac10db03efe2b76cdbed89277f8d Mon Sep 17 00:00:00 2001 From: maxmindlin <35264981+maxmindlin@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:08:57 -0400 Subject: [PATCH] http output --- Cargo.lock | 608 +------------------ scout-worker/Cargo.toml | 4 +- scout-worker/src/config.rs | 28 +- scout-worker/src/http/mod.rs | 2 + scout-worker/src/http/sender.rs | 35 ++ scout-worker/src/{http.rs => http/server.rs} | 43 +- scout-worker/src/main.rs | 41 +- scout-worker/src/output.rs | 21 + scout-worker/src/rmq/consumer.rs | 16 +- scout-worker/src/rmq/producer.rs | 8 +- 10 files changed, 181 insertions(+), 625 deletions(-) create mode 100644 scout-worker/src/http/mod.rs create mode 100644 scout-worker/src/http/sender.rs rename scout-worker/src/{http.rs => http/server.rs} (52%) create mode 100644 scout-worker/src/output.rs diff --git a/Cargo.lock b/Cargo.lock index 8d71059..baa42e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,17 +199,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.11" @@ -330,45 +319,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "asn1-rs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-channel" version = "2.3.1" @@ -543,33 +493,6 @@ dependencies = [ "arrayvec", ] -[[package]] -name = "aws-lc-rs" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" -dependencies = [ - "aws-lc-sys", - "mirai-annotations", - "paste", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", - "libc", - "paste", -] - [[package]] name = "backtrace" version = "0.3.71" @@ -603,35 +526,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.5.0", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", - "which", -] - [[package]] name = "bit_field" version = "0.10.2" @@ -665,15 +559,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "blocking" version = "1.6.1" @@ -753,15 +638,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cc" version = "1.0.97" @@ -773,15 +649,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-expr" version = "0.15.8" @@ -798,27 +665,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clipboard-win" version = "4.5.0" @@ -830,27 +676,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - -[[package]] -name = "cms" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" -dependencies = [ - "const-oid", - "der", - "spki", - "x509-cert", -] - [[package]] name = "color_quant" version = "1.1.0" @@ -866,12 +691,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "convert_case" version = "0.4.0" @@ -998,50 +817,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - -[[package]] -name = "der" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" -dependencies = [ - "const-oid", - "der_derive", - "flagset", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "der-parser" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "der_derive" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "deranged" version = "0.3.11" @@ -1064,15 +839,6 @@ dependencies = [ "syn", ] -[[package]] -name = "des" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" -dependencies = [ - "cipher", -] - [[package]] name = "digest" version = "0.10.7" @@ -1081,7 +847,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1105,29 +870,12 @@ dependencies = [ "winapi", ] -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dunce" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" - [[package]] name = "either" version = "1.12.0" @@ -1293,12 +1041,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "flagset" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec" - [[package]] name = "flate2" version = "1.0.30" @@ -1350,12 +1092,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.30" @@ -1516,12 +1252,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "h2" version = "0.3.26" @@ -1594,30 +1324,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "http" version = "0.2.12" @@ -1739,7 +1445,7 @@ dependencies = [ "hyper 0.14.28", "log", "rustls 0.20.9", - "rustls-native-certs 0.6.3", + "rustls-native-certs", "tokio", "tokio-rustls 0.23.4", ] @@ -1754,7 +1460,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.11", + "rustls 0.23.5", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -1879,16 +1585,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "instant" version = "0.1.13" @@ -1999,12 +1695,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lebe" version = "0.5.2" @@ -2028,16 +1718,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "libloading" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" -dependencies = [ - "cfg-if", - "windows-targets 0.52.5", -] - [[package]] name = "libredox" version = "0.1.3" @@ -2161,12 +1841,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "native-tls" version = "0.2.11" @@ -2304,15 +1978,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "oid-registry" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" -dependencies = [ - "asn1-rs", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -2363,28 +2028,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "p12-keystore" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7b60d0b2dcace322e6e8c4499c4c8bdf331c1bae046a54be5e4191c3610286" -dependencies = [ - "cbc", - "cms", - "der", - "des", - "hex", - "hmac", - "pkcs12", - "pkcs5", - "rand", - "rc2", - "sha1", - "sha2", - "thiserror", - "x509-parser", -] - [[package]] name = "parking" version = "2.2.0" @@ -2420,25 +2063,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -2500,36 +2124,6 @@ dependencies = [ "futures-io", ] -[[package]] -name = "pkcs12" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" -dependencies = [ - "cms", - "const-oid", - "der", - "digest", - "spki", - "x509-cert", - "zeroize", -] - -[[package]] -name = "pkcs5" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" -dependencies = [ - "aes", - "cbc", - "der", - "pbkdf2", - "scrypt", - "sha2", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.30" @@ -2590,16 +2184,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettyplease" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.82" @@ -2778,15 +2362,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rc2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" -dependencies = [ - "cipher", -] - [[package]] name = "reactor-trait" version = "1.1.0" @@ -2943,12 +2518,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc_version" version = "0.4.0" @@ -2958,15 +2527,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - [[package]] name = "rustix" version = "0.37.13" @@ -3008,12 +2568,10 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "afabcee0551bd1aa3e18e5adbf2c0544722014b899adb31bd186ec638d3da97e" dependencies = [ - "aws-lc-rs", - "log", "once_cell", "rustls-pki-types", "rustls-webpki", @@ -3021,19 +2579,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-connector" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a826801254b6cfcd2508a0508c01b7c1bca21d3673e84d86da084781b83d5" -dependencies = [ - "log", - "rustls 0.23.11", - "rustls-native-certs 0.7.1", - "rustls-pki-types", - "rustls-webpki", -] - [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -3046,19 +2591,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.1.2", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3090,7 +2622,6 @@ version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ - "aws-lc-rs", "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", @@ -3126,15 +2657,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "salsa20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" -dependencies = [ - "cipher", -] - [[package]] name = "schannel" version = "0.1.23" @@ -3190,11 +2712,13 @@ dependencies = [ "actix-web", "futures-lite 2.3.0", "lapin", + "reqwest", "scout-interpreter", "serde", "serde_json", "tokio", "toml", + "tracing", ] [[package]] @@ -3213,17 +2737,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "scrypt" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" -dependencies = [ - "pbkdf2", - "salsa20", - "sha2", -] - [[package]] name = "sct" version = "0.7.1" @@ -3326,23 +2839,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -3417,16 +2913,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - [[package]] name = "str-buf" version = "1.0.6" @@ -3456,17 +2942,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "system-configuration" version = "0.5.1" @@ -3514,8 +2989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" dependencies = [ "cfg-if", - "p12-keystore", - "rustls-connector", + "native-tls", "rustls-pemfile 2.1.2", ] @@ -3698,7 +3172,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.11", + "rustls 0.23.5", "rustls-pki-types", "tokio", ] @@ -3785,9 +3259,21 @@ checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -4041,18 +3527,6 @@ dependencies = [ "cc", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.34", -] - [[package]] name = "winapi" version = "0.3.9" @@ -4290,34 +3764,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "x509-cert" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" -dependencies = [ - "const-oid", - "der", - "spki", -] - -[[package]] -name = "x509-parser" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "rusticata-macros", - "thiserror", - "time", -] - [[package]] name = "zerocopy" version = "0.7.35" @@ -4343,20 +3789,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] [[package]] name = "zstd" diff --git a/scout-worker/Cargo.toml b/scout-worker/Cargo.toml index bd3ee16..ef11991 100644 --- a/scout-worker/Cargo.toml +++ b/scout-worker/Cargo.toml @@ -9,6 +9,8 @@ actix-web = "4" tokio = { version = "1", features = ["full"] } serde = { version = "1.0.203", features = ["derive"] } toml = "0.8.16" -lapin = "2.5.0" +lapin = { version = "2.5.0", default-features = false, features = ["native-tls"] } futures-lite = "2.3.0" serde_json = "1.0" +tracing = "0.1.40" +reqwest = "0.12.5" \ No newline at end of file diff --git a/scout-worker/src/config.rs b/scout-worker/src/config.rs index dfb8159..f1abd24 100644 --- a/scout-worker/src/config.rs +++ b/scout-worker/src/config.rs @@ -15,20 +15,34 @@ pub struct Config { #[derive(Debug, Default, Deserialize)] pub struct ConfigOutputs { pub rmq: Option, + pub http: Option, } #[derive(Debug, Default, Deserialize)] pub struct ConfigInputs { - pub http: Option, + pub http: Option, pub rmq: Option, } #[derive(Debug, Deserialize)] -pub struct ConfigInputsHttp { +pub struct ConfigInputHttp { pub addr: String, pub port: usize, } +#[derive(Debug, Deserialize)] +pub enum OutputMethods { + POST, + PUT, + PATCH, +} + +#[derive(Debug, Deserialize)] +pub struct ConfigOutputHttp { + pub endpoint: String, + pub method: OutputMethods, +} + #[derive(Debug, Deserialize)] pub struct ConfigRMQ { pub addr: String, @@ -45,3 +59,13 @@ impl Config { toml::from_str(&content).map_err(|e| WorkerError::ConfigError(e.to_string())) } } + +impl std::fmt::Display for OutputMethods { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::PATCH => write!(f, "PATCH"), + Self::POST => write!(f, "POST"), + Self::PUT => write!(f, "PUT"), + } + } +} diff --git a/scout-worker/src/http/mod.rs b/scout-worker/src/http/mod.rs new file mode 100644 index 0000000..587a0dc --- /dev/null +++ b/scout-worker/src/http/mod.rs @@ -0,0 +1,2 @@ +pub mod sender; +pub mod server; diff --git a/scout-worker/src/http/sender.rs b/scout-worker/src/http/sender.rs new file mode 100644 index 0000000..3712866 --- /dev/null +++ b/scout-worker/src/http/sender.rs @@ -0,0 +1,35 @@ +use reqwest::Method; + +use crate::config::OutputMethods; + +pub struct Sender { + client: reqwest::Client, + method: Method, + endpoint: String, +} + +impl Sender { + pub fn new( + method: &OutputMethods, + endpoint: String, + ) -> Result> { + let client = reqwest::Client::new(); + let method = Method::from_bytes(method.to_string().as_bytes())?; + Ok(Self { + client, + method, + endpoint, + }) + } + + pub async fn send(&self, payload: &str) -> Result<(), reqwest::Error> { + let req = self + .client + .request(self.method.clone(), &self.endpoint) + .body(payload.to_owned()) + .build()?; + self.client.execute(req).await?; + + Ok(()) + } +} diff --git a/scout-worker/src/http.rs b/scout-worker/src/http/server.rs similarity index 52% rename from scout-worker/src/http.rs rename to scout-worker/src/http/server.rs index 1ef444a..f5e6926 100644 --- a/scout-worker/src/http.rs +++ b/scout-worker/src/http/server.rs @@ -1,15 +1,26 @@ use std::{ fs, io::{self, ErrorKind}, + sync::Arc, }; -use actix_web::{get, http::StatusCode, post, web, App, HttpResponse, HttpServer, Responder}; +use actix_web::{ + get, + http::StatusCode, + post, + web::{self, Data}, + App, HttpResponse, HttpServer, Responder, +}; use scout_interpreter::builder::InterpreterBuilder; +use tracing::info; -use crate::{config::ConfigInputsHttp, models::incoming}; +use crate::{config::ConfigInputHttp, models::incoming, Output}; #[post("/")] -async fn crawl(body: web::Json) -> impl Responder { +async fn crawl( + outputs: Data>>, + body: web::Json, +) -> impl Responder { match fs::read_to_string(&body.file) { Ok(content) => { let interpreter = InterpreterBuilder::default().build().await.unwrap(); @@ -21,6 +32,12 @@ async fn crawl(body: web::Json) -> impl Responder { let payload = res.lock().await.to_json(); interpreter.close().await; + for output in outputs.iter() { + if let Err(e) = output.send(&payload).await { + println!("error sending to output: {e}"); + } + } + HttpResponse::build(StatusCode::OK) .content_type("application/json") .body(payload) @@ -37,9 +54,19 @@ async fn health() -> impl Responder { "OK" } -pub async fn start_http_consumer(config: &ConfigInputsHttp) -> Result<(), io::Error> { - HttpServer::new(move || App::new().service(crawl).service(health)) - .bind((config.addr.as_str(), config.port as u16))? - .run() - .await +pub async fn start_http_consumer( + config: &ConfigInputHttp, + outputs: Arc>, +) -> Result<(), io::Error> { + info!("starting HTTP server on {}:{}", config.addr, config.port); + let data = Data::new(outputs.clone()); + HttpServer::new(move || { + App::new() + .app_data(data.clone()) + .service(crawl) + .service(health) + }) + .bind((config.addr.as_str(), config.port as u16))? + .run() + .await } diff --git a/scout-worker/src/main.rs b/scout-worker/src/main.rs index 1aeeb8f..5452411 100644 --- a/scout-worker/src/main.rs +++ b/scout-worker/src/main.rs @@ -1,30 +1,21 @@ +use std::sync::Arc; + use config::Config; +use http::sender::Sender; +use output::Output; use rmq::producer::Producer; +use tracing::error; mod config; mod http; mod models; +mod output; mod rmq; pub enum WorkerError { ConfigError(String), } -pub enum Output { - RMQ(rmq::producer::Producer), -} - -impl Output { - pub async fn send(&self, payload: String) -> Result<(), Box> { - match self { - Output::RMQ(p) => { - p.send(payload).await?; - Ok(()) - } - } - } -} - async fn start(config: Config) -> Result<(), Box> { let mut outputs = Vec::new(); if let Some(outputs_config) = config.outputs { @@ -32,13 +23,21 @@ async fn start(config: Config) -> Result<(), Box> { let rmq_out = Producer::new(&rmq).await?; outputs.push(Output::RMQ(rmq_out)); } + + if let Some(http) = outputs_config.http { + let http_out = Sender::new(&http.method, http.endpoint)?; + outputs.push(Output::HTTP(http_out)); + } } + let aoutputs = Arc::new(outputs); if let Some(http_config) = config.inputs.http { - http::start_http_consumer(&http_config).await?; + http::server::start_http_consumer(&http_config, aoutputs.clone()).await?; } else if let Some(rmq_config) = config.inputs.rmq { - let mut consumer = rmq::consumer::Consumer::new(&rmq_config).await?; - consumer.start().await.expect("error starting consumer"); + rmq::consumer::Consumer::new(&rmq_config, aoutputs.clone()) + .await? + .start() + .await?; } Ok(()) @@ -46,8 +45,12 @@ async fn start(config: Config) -> Result<(), Box> { #[tokio::main] async fn main() { + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "info"); + } + let config = Config::load_file(None).unwrap_or_default(); if let Err(e) = start(config).await { - println!("error starting worker: {e}") + error!("error starting worker: {e}"); } } diff --git a/scout-worker/src/output.rs b/scout-worker/src/output.rs new file mode 100644 index 0000000..e1f0977 --- /dev/null +++ b/scout-worker/src/output.rs @@ -0,0 +1,21 @@ +use crate::{http, rmq}; + +pub enum Output { + RMQ(rmq::producer::Producer), + HTTP(http::sender::Sender), +} + +impl Output { + pub async fn send(&self, payload: &str) -> Result<(), Box> { + match self { + Output::RMQ(p) => { + p.send(payload).await?; + Ok(()) + } + Output::HTTP(sender) => { + sender.send(payload).await?; + Ok(()) + } + } + } +} diff --git a/scout-worker/src/rmq/consumer.rs b/scout-worker/src/rmq/consumer.rs index eb85f27..49b0261 100644 --- a/scout-worker/src/rmq/consumer.rs +++ b/scout-worker/src/rmq/consumer.rs @@ -11,9 +11,10 @@ use scout_interpreter::{ builder::{BuilderError, InterpreterBuilder}, Interpreter, InterpreterError, }; -use std::{fmt::Display, fs, str}; +use std::{fmt::Display, fs, str, sync::Arc}; +use tracing::{error, info}; -use crate::{config::ConfigRMQ, models::incoming}; +use crate::{config::ConfigRMQ, models::incoming, Output}; #[derive(Debug)] pub enum ConsumerError { @@ -25,10 +26,11 @@ pub struct Consumer { chann: Channel, queue: String, interpreter: Interpreter, + outputs: Arc>, } impl Consumer { - pub async fn new(config: &ConfigRMQ) -> Result { + pub async fn new(config: &ConfigRMQ, outputs: Arc>) -> Result { let conn = Connection::connect(&config.addr, ConnectionProperties::default()).await?; let chann = conn.create_channel().await?; let interpreter = InterpreterBuilder::default().build().await?; @@ -64,6 +66,7 @@ impl Consumer { chann, queue: config.queue.clone(), interpreter, + outputs, }) } @@ -90,10 +93,17 @@ impl Consumer { ) .await?; + info!("listening on queue {}", self.queue); while let Some(delivery) = consumer.next().await { if let Ok(delivery) = delivery { + info!("processing msg"); match self.process(&delivery.data).await { Ok(res) => { + for output in self.outputs.iter() { + if let Err(e) = output.send(&res).await { + error!("error sending to output: {e}"); + } + } delivery.ack(BasicAckOptions::default()).await?; } Err(_) => { diff --git a/scout-worker/src/rmq/producer.rs b/scout-worker/src/rmq/producer.rs index 8b2f5ff..f3ef99c 100644 --- a/scout-worker/src/rmq/producer.rs +++ b/scout-worker/src/rmq/producer.rs @@ -5,6 +5,7 @@ use lapin::{ types::FieldTable, BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind, }; +use tracing::info; use crate::config::ConfigRMQ; @@ -15,7 +16,6 @@ pub enum ProducerError { pub struct Producer { chann: Channel, - queue: String, exchange: String, out_key: String, } @@ -44,19 +44,19 @@ impl Producer { Ok(Self { chann, - queue: config.queue.clone(), exchange: config.exchange.clone(), out_key: config.routing_key.clone(), }) } - pub async fn send(&self, payload: String) -> Result<(), ProducerError> { + pub async fn send(&self, payload: &str) -> Result<(), ProducerError> { + info!("publishing message to {}", self.out_key); self.chann .basic_publish( &self.exchange, &self.out_key, BasicPublishOptions::default(), - &payload, + &payload.as_bytes(), BasicProperties::default(), ) .await?;