From 0d68e19de3cf1802aebd8be2e75ea3d3617ec91f Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Thu, 5 Oct 2023 12:57:12 -0400 Subject: [PATCH 01/15] add test dependencies, update lockfile --- Cargo.lock | 843 +++++++++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 4 + 2 files changed, 723 insertions(+), 124 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 147c4b3..9d3f8f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,39 +4,53 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -52,14 +66,29 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", ] +[[package]] +name = "assert_cmd" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -68,9 +97,46 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bollard-stubs" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +dependencies = [ + "chrono", + "serde", + "serde_with", +] + +[[package]] +name = "bstr" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" @@ -80,9 +146,12 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -90,47 +159,66 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "clap" -version = "4.3.0" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.0" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", - "bitflags", "clap_lex", "strsim", ] [[package]] name = "clap_derive" -version = "4.3.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.37", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "colorchoice" @@ -151,6 +239,95 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -178,9 +355,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -197,6 +374,122 @@ dependencies = [ "libc", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + [[package]] name = "hashbrown" version = "0.14.0" @@ -211,18 +504,24 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] [[package]] name = "humantime" @@ -230,6 +529,35 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" version = "2.0.0" @@ -247,26 +575,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] -name = "io-lifetimes" -version = "1.0.11" +name = "is-terminal" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "libc", + "hermit-abi", + "rustix", "windows-sys 0.48.0", ] [[package]] -name = "is-terminal" -version = "0.4.7" +name = "itertools" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.48.0", + "either", ] [[package]] @@ -275,6 +600,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -283,40 +617,42 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "log" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "nektar" version = "0.0.2" dependencies = [ + "assert_cmd", "clap", "console", "env_logger", "log", - "ordered-float 3.7.0", + "ordered-float 3.9.1", "serde", "serde_json", "serde_yaml", + "testcontainers", "thiserror", "thrift", "try_from", @@ -324,28 +660,28 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "once_cell" -version = "1.17.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-float" @@ -358,36 +694,124 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.7.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "num-traits", ] +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" +dependencies = [ + "anstyle", + "difflib", + "itertools", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.31" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "regex" -version = "1.8.3" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", @@ -396,19 +820,18 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rustix" -version = "0.37.19" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys 0.48.0", @@ -422,35 +845,57 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "serde" -version = "1.0.174" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.174" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.37", ] [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_yaml" version = "0.9.25" @@ -464,17 +909,54 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" -version = "2.0.27" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -483,31 +965,54 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "testcontainers" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e2b1567ca8a2b819ea7b28c92be35d9f76fb9edb214321dcc86eb96023d1f87" +dependencies = [ + "bollard-stubs", + "futures", + "hex", + "hmac", + "log", + "rand", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.37", ] [[package]] @@ -538,20 +1043,26 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unsafe-libyaml" @@ -565,6 +1076,81 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + [[package]] name = "winapi" version = "0.3.9" @@ -583,9 +1169,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -596,6 +1182,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -611,7 +1206,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.5", ] [[package]] @@ -631,17 +1226,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -652,9 +1247,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -664,9 +1259,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -676,9 +1271,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -688,9 +1283,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -700,9 +1295,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -712,9 +1307,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -724,6 +1319,6 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/Cargo.toml b/Cargo.toml index 0b20853..5555a89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,10 @@ console = "0.15.7" serde = { version = "1.0.174", features = ["derive"] } serde_json = "1.0.103" serde_yaml = { version = "0.9.25", optional = true } +assert_cmd = "2.0.12" + +[dev-dependencies] +testcontainers = "0.14.0" [features] default = ["yaml"] From e3e2e538c7f20bda6e2a048ffd76465e6dd96d1a Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Thu, 5 Oct 2023 16:16:37 -0400 Subject: [PATCH 02/15] working test harness with tokio + testcontainers --- Cargo.lock | 121 ++++++++++++++++++++++++++++++ Cargo.toml | 5 ++ tests/integration_test.rs | 152 ++++++++++++++++++++++++++++++++++++++ tests/predicates/json.rs | 18 +++++ tests/predicates/mod.rs | 3 + tests/predicates/yaml.rs | 18 +++++ 6 files changed, 317 insertions(+) create mode 100644 tests/integration_test.rs create mode 100644 tests/predicates/json.rs create mode 100644 tests/predicates/mod.rs create mode 100644 tests/predicates/yaml.rs diff --git a/Cargo.lock b/Cargo.lock index 9d3f8f9..2769873 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.1.1" @@ -89,12 +104,33 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async_once" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82" + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "2.4.0" @@ -264,6 +300,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" +dependencies = [ + "quote", + "syn 2.0.37", +] + [[package]] name = "darling" version = "0.13.4" @@ -374,6 +420,15 @@ dependencies = [ "libc", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -490,6 +545,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "hashbrown" version = "0.14.0" @@ -639,25 +700,45 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "nektar" version = "0.0.2" dependencies = [ "assert_cmd", + "async_once", "clap", "console", + "ctor", "env_logger", + "lazy_static", "log", "ordered-float 3.9.1", + "predicates", "serde", "serde_json", "serde_yaml", "testcontainers", "thiserror", "thrift", + "tokio", "try_from", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "num-traits" version = "0.2.16" @@ -677,6 +758,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -727,8 +817,11 @@ checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" dependencies = [ "anstyle", "difflib", + "float-cmp", "itertools", + "normalize-line-endings", "predicates-core", + "regex", ] [[package]] @@ -824,6 +917,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustix" version = "0.38.14" @@ -1037,6 +1136,28 @@ dependencies = [ "threadpool", ] +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "try_from" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index 5555a89..734ceb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,12 @@ serde_yaml = { version = "0.9.25", optional = true } assert_cmd = "2.0.12" [dev-dependencies] +async_once = "0.2.6" +ctor = "0.2.5" +lazy_static = "1.4.0" +predicates = "3.0.4" testcontainers = "0.14.0" +tokio = { version = "1.32.0", features = ["macros", "sync", "rt"] } [features] default = ["yaml"] diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 0000000..ae07050 --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,152 @@ +mod predicates; + +use predicates::json::IsJson; + +use assert_cmd::Command; +use ctor::{ctor, dtor}; +use lazy_static::lazy_static; +use std::{future::Future, thread}; +use testcontainers::clients::Cli; +use testcontainers::core::WaitFor; +use testcontainers::images::generic::GenericImage; +use tokio::{ + runtime, + sync::{ + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Mutex, + }, +}; + +const METASTORE_HOST: &str = "localhost"; +const METASTORE_PORT: u16 = 9083; +const HIVE_VERSION: &str = "4.0.0-beta-2-SNAPSHOT"; +const SERVICE_NAME_VAR: &str = "SERVICE_NAME"; +const SERVICE_NAME: &str = "metastore"; +const SERVICE_OPTS_VAR: &str = "SERVICE_OPTS"; +const SERVICE_OPTS: &str = "-Dhive.root.logger=console"; +const READY_MSG: &str = "Successfully created a default database with name: default"; + +/// Channels for communication with container started on thread +struct Channel { + tx: UnboundedSender, + rx: Mutex>, +} + +/// Channel constructor +fn channel() -> Channel { + let (tx, rx) = mpsc::unbounded_channel(); + Channel { + tx, + rx: Mutex::new(rx), + } +} + +/// Messages for communicating with container +#[derive(Debug)] +enum ContainerCommands { + FetchPort, + Stop, +} + +lazy_static! { + static ref METASTORE_IN: Channel = channel(); + static ref METASTORE_HOST_PORT: Channel = channel(); + static ref METASTORE_STOPPED: Channel<()> = channel(); +} + +/// Start testcontainers, open channels for communication +/// +/// Necessary to support starting one long lived testcontainer +/// for all test methods. +async fn set_up() { + let docker = Cli::default(); + let msg_await = WaitFor::message_on_stderr(READY_MSG.to_string()); + let hive_image = GenericImage::new("apache/hive", HIVE_VERSION) + .with_wait_for(msg_await) + .with_exposed_port(METASTORE_PORT) + .with_env_var(SERVICE_NAME_VAR, SERVICE_NAME) + .with_env_var(SERVICE_OPTS_VAR, SERVICE_OPTS); + let hive_metastore = docker.run(hive_image); + let open_port = hive_metastore.ports().map_to_host_port_ipv4(METASTORE_PORT); + + log::info!("{}:{}", METASTORE_HOST, METASTORE_PORT); + let mut rx = METASTORE_IN.rx.lock().await; + while let Some(command) = rx.recv().await { + println!("Received container command: {:?}", command); + match command { + ContainerCommands::FetchPort => { + METASTORE_HOST_PORT.tx.send(open_port.unwrap()).unwrap() + } + ContainerCommands::Stop => { + hive_metastore.stop(); + METASTORE_STOPPED.tx.send(()).unwrap(); + rx.close(); + } + } + } +} + +/// Helper function for blocking run, used to start testcontainer +fn execute_blocking(f: F) { + runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(f); +} + +#[ctor] +/// Start container on process start +fn on_startup() { + thread::spawn(|| execute_blocking(set_up())); +} + +#[dtor] +/// Kill container on process end +fn on_shutdown() { + execute_blocking(clean_up()); +} + +/// Destructor, kill container and notify channels +async fn clean_up() { + METASTORE_IN.tx.send(ContainerCommands::Stop).unwrap(); + METASTORE_STOPPED.rx.lock().await.recv().await; +} + +// BEGIN: test cases + +#[tokio::test] +async fn test_get_default_catalog() { + METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); + let mut cmd = Command::cargo_bin("nektar").unwrap(); + let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); + cmd.arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg("get-catalog") + .arg("hive") + .assert() + .success(); +} + +#[tokio::test] +async fn test_create_and_get_catalog() { + METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); + let mut create_cmd = Command::cargo_bin("nektar").unwrap(); + let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); + let catalog_name = "test"; + create_cmd + .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg("create-catalog") + .arg(catalog_name) + .arg("a description") + .arg("file:/opt/hive/data/warehouse") + .assert() + .success(); + + let mut get_cmd = Command::cargo_bin("nektar").unwrap(); + get_cmd + .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg("get-catalog") + .arg(catalog_name) + .assert() + .stdout(IsJson) + .success(); +} diff --git a/tests/predicates/json.rs b/tests/predicates/json.rs new file mode 100644 index 0000000..dad7b47 --- /dev/null +++ b/tests/predicates/json.rs @@ -0,0 +1,18 @@ +use predicates::prelude::*; +use serde_json::Value as JsonValue; +use std::fmt; + +pub struct IsJson; + +impl predicates::reflection::PredicateReflection for IsJson {} + +impl Predicate for IsJson { + fn eval(&self, variable: &str) -> bool { + serde_json::from_str::(&variable).is_ok() + } +} +impl fmt::Display for IsJson { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "var.is_json()") + } +} diff --git a/tests/predicates/mod.rs b/tests/predicates/mod.rs new file mode 100644 index 0000000..e84fd93 --- /dev/null +++ b/tests/predicates/mod.rs @@ -0,0 +1,3 @@ +pub mod json; +pub mod yaml; + diff --git a/tests/predicates/yaml.rs b/tests/predicates/yaml.rs new file mode 100644 index 0000000..d1f7183 --- /dev/null +++ b/tests/predicates/yaml.rs @@ -0,0 +1,18 @@ +use predicates::prelude::*; +use serde_yaml::Value as YamlValue; +use std::fmt; + +pub struct IsYaml; + +impl predicates::reflection::PredicateReflection for IsYaml {} + +impl Predicate for IsYaml { + fn eval(&self, variable: &str) -> bool { + serde_yaml::from_str::(&variable).is_ok() + } +} +impl fmt::Display for IsYaml { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "var.is_json()") + } +} From cec82891a9f88df8d721ed9a0bdcca4356fe263e Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 13:03:39 -0400 Subject: [PATCH 03/15] only run build on PR --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce71a16..16763b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ name: build -on: [push, pull_request] +on: [pull_request] env: CARGO_TERM_COLOR: always From de845644650e3831a189a8e5719f1d55c4893133 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 13:17:55 -0400 Subject: [PATCH 04/15] make description optional --- src/cmds/catalogs.rs | 3 ++- tests/integration_test.rs | 3 ++- tests/predicates/mod.rs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cmds/catalogs.rs b/src/cmds/catalogs.rs index b23278e..c76a824 100644 --- a/src/cmds/catalogs.rs +++ b/src/cmds/catalogs.rs @@ -41,8 +41,9 @@ impl RunCommand for GetCatalogs { #[derive(Debug, Args)] pub struct CreateCatalog { name: String, - description: Option, location_uri: Option, + #[arg(long = "description", short = 'd')] + description: Option, } impl Into for CreateCatalog { diff --git a/tests/integration_test.rs b/tests/integration_test.rs index ae07050..1ec7e8d 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -136,8 +136,9 @@ async fn test_create_and_get_catalog() { .arg(format!("{}:{}", METASTORE_HOST, open_port)) .arg("create-catalog") .arg(catalog_name) - .arg("a description") .arg("file:/opt/hive/data/warehouse") + .arg("-d") + .arg("a description") .assert() .success(); diff --git a/tests/predicates/mod.rs b/tests/predicates/mod.rs index e84fd93..b94867c 100644 --- a/tests/predicates/mod.rs +++ b/tests/predicates/mod.rs @@ -1,3 +1,3 @@ pub mod json; +#[cfg(feature = "yaml")] pub mod yaml; - From 5fef50d119bdb097d04619be0a5e1ad501d28179 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 13:18:39 -0400 Subject: [PATCH 05/15] make location non-optional --- src/cmds/catalogs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cmds/catalogs.rs b/src/cmds/catalogs.rs index c76a824..fd70ca8 100644 --- a/src/cmds/catalogs.rs +++ b/src/cmds/catalogs.rs @@ -41,7 +41,7 @@ impl RunCommand for GetCatalogs { #[derive(Debug, Args)] pub struct CreateCatalog { name: String, - location_uri: Option, + location_uri: String, #[arg(long = "description", short = 'd')] description: Option, } @@ -50,8 +50,8 @@ impl Into for CreateCatalog { fn into(self) -> Catalog { Catalog { name: Some(self.name), + location_uri: Some(self.location_uri), description: self.description, - location_uri: self.location_uri, } } } From b318823d85e4dbd5d7d7f4bf7fa5e4db40f309b1 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 15:13:24 -0400 Subject: [PATCH 06/15] working create table method --- resources/test/table.json | 75 +++++++++++++++++++++++++++++++++++++++ src/cli.rs | 6 +++- src/cmds/tables.rs | 36 ++++++++++++++++++- src/hive_metastore.rs | 26 +++++++------- tests/integration_test.rs | 26 ++++++++++++++ 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 resources/test/table.json diff --git a/resources/test/table.json b/resources/test/table.json new file mode 100644 index 0000000..06d5c7c --- /dev/null +++ b/resources/test/table.json @@ -0,0 +1,75 @@ +{ + "table_name": "artist_affinity_28days", + "db_name": "default", + "owner": "regadas", + "sd": { + "cols": [ + { + "name": "user_id", + "type_": "string", + "comment": "" + }, + { + "name": "artist_gid", + "type_": "string", + "comment": "" + }, + { + "name": "days", + "type_": "bigint", + "comment": "" + }, + { + "name": "affinity", + "type_": "double", + "comment": "" + } + ], + "location": "file:/opt/hive/data/warehouse", + "input_format": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "output_format": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "compressed": false, + "num_buckets": 0, + "serde_info": { + "name": "artist_affinity_28days", + "serialization_lib": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", + "parameters": {}, + "description": null, + "serializer_class": null, + "deserializer_class": null, + "serde_type": null + }, + "bucket_cols": [], + "sort_cols": [], + "parameters": {}, + "skewed_info": { + "skewed_col_names": [], + "skewed_col_values": [], + "skewed_col_value_location_maps": {} + }, + "stored_as_sub_directories": false + }, + "partition_keys": [ + { + "name": "ds", + "type_": "date", + "comment": "" + } + ], + "parameters": { + "STATS_GENERATED_VIA_STATS_TASK": "workaround for potential lack of HIVE-12730", + "auto.purge": "false", + "presto_query_id": "20230228_212728_00112_p3e7g", + "presto_version": "403", + "transient_lastDdlTime": "1677619649" + }, + "view_original_text": "", + "view_expanded_text": "", + "table_type": "MANAGED_TABLE", + "privileges": null, + "temporary": null, + "rewrite_enabled": false, + "creation_metadata": null, + "cat_name": "hive", + "owner_type": 1 +} diff --git a/src/cli.rs b/src/cli.rs index 0d23370..06f7208 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -13,7 +13,7 @@ use crate::cmds::{ catalogs::{CreateCatalog, GetCatalog}, databases::GetDatabases, partitions::{GetPartitionNamesByParts, GetPartitions}, - tables::GetTable, + tables::{CreateTable, GetTable}, }; use crate::error::CliError; @@ -54,6 +54,7 @@ pub enum Commands { GetPartitionNamesByParts(GetPartitionNamesByParts), GetDatabases(GetDatabases), CreateCatalog(CreateCatalog), + CreateTable(CreateTable), } fn serialize(f: Format, v: T) -> Result { @@ -92,6 +93,9 @@ impl Cli { Commands::GetCatalogs(get_catalogs) => { serialize(self.format, get_catalogs.run(client)?) } + Commands::CreateTable(create_table) => { + serialize(self.format, create_table.run(client)?) + } } } } diff --git a/src/cmds/tables.rs b/src/cmds/tables.rs index b1a9fe5..173178f 100644 --- a/src/cmds/tables.rs +++ b/src/cmds/tables.rs @@ -1,8 +1,15 @@ use crate::cli::MetastoreClient; use crate::cli::RunCommand; use crate::error::CliError; -use clap::Args; +use clap::{Args, ValueEnum}; use nektar::{TThriftHiveMetastoreSyncClient, Table}; +use std::convert::TryInto; +use std::io::prelude::*; +use std::path::PathBuf; +use std::{ + fs::File, + time::{SystemTime, UNIX_EPOCH}, +}; #[derive(Debug, Args)] pub struct GetTable { @@ -15,3 +22,30 @@ impl RunCommand> for GetTable { Ok(client.get_table_objects_by_name(self.database, self.table)?) } } + +#[derive(ValueEnum, Debug, Clone)] +pub enum TableType { + External, + Internal, +} + +#[derive(Debug, Args)] +pub struct CreateTable { + table_definition_file: PathBuf, +} + +impl RunCommand for CreateTable { + fn run(self, mut client: MetastoreClient) -> Result { + let now = SystemTime::now(); + let since_epoch: i32 = now + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() + .try_into() + .unwrap(); + let file = File::open(self.table_definition_file).unwrap(); + let mut table: Table = serde_json::from_reader(file).unwrap(); + table.create_time = Some(since_epoch); + Ok(client.create_table(table.clone())?).map(|_| table) + } +} diff --git a/src/hive_metastore.rs b/src/hive_metastore.rs index 7b89d27..eb522d9 100644 --- a/src/hive_metastore.rs +++ b/src/hive_metastore.rs @@ -27,7 +27,7 @@ use thrift::protocol::verify_expected_sequence_number; use thrift::protocol::verify_expected_service_call; use thrift::protocol::verify_required_field_exists; use thrift::server::TProcessor; -use serde::Serialize; +use serde::{Serialize, Deserialize}; use crate::fb303; @@ -91,7 +91,7 @@ impl From<&HiveObjectType> for i32 { } } -#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct PrincipalType(pub i32); impl PrincipalType { @@ -631,7 +631,7 @@ impl From<&EventRequestType> for i32 { } } -#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct SerdeType(pub i32); impl SerdeType { @@ -1290,7 +1290,7 @@ impl TSerializable for Version { // FieldSchema // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct FieldSchema { pub name: Option, pub type_: Option, @@ -2642,7 +2642,7 @@ impl TSerializable for HiveObjectRef { // PrivilegeGrantInfo // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct PrivilegeGrantInfo { pub privilege: Option, pub create_time: Option, @@ -2927,7 +2927,7 @@ impl TSerializable for PrivilegeBag { // PrincipalPrivilegeSet // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct PrincipalPrivilegeSet { pub user_privileges: Option>>, pub group_privileges: Option>>, @@ -3882,7 +3882,7 @@ impl TSerializable for GrantRevokeRoleResponse { // Catalog // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct Catalog { pub name: Option, pub description: Option, @@ -4490,7 +4490,7 @@ impl TSerializable for Database { // SerDeInfo // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct SerDeInfo { pub name: Option, pub serialization_lib: Option, @@ -4637,7 +4637,7 @@ impl TSerializable for SerDeInfo { // Order // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct Order { pub col: Option, pub order: Option, @@ -4707,7 +4707,7 @@ impl TSerializable for Order { // SkewedInfo // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct SkewedInfo { pub skewed_col_names: Option>, pub skewed_col_values: Option>>, @@ -4842,7 +4842,7 @@ impl TSerializable for SkewedInfo { // StorageDescriptor // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct StorageDescriptor { pub cols: Option>, pub location: Option, @@ -5084,7 +5084,7 @@ impl TSerializable for StorageDescriptor { // Table // -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct Table { pub table_name: Option, pub db_name: Option, @@ -13935,7 +13935,7 @@ impl TSerializable for BasicTxnInfo { // CreationMetadata // -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct CreationMetadata { pub cat_name: String, pub db_name: String, diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 1ec7e8d..a19aeff 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -151,3 +151,29 @@ async fn test_create_and_get_catalog() { .stdout(IsJson) .success(); } + +#[tokio::test] +async fn test_create_table() { + METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); + let mut create_cmd = Command::cargo_bin("nektar").unwrap(); + let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); + let catalog_name = "test"; + create_cmd + .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg("create-catalog") + .arg(catalog_name) + .arg("file:/opt/hive/data/warehouse") + .arg("-d") + .arg("a description") + .assert() + .success(); + + let mut get_cmd = Command::cargo_bin("nektar").unwrap(); + get_cmd + .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg("get-catalog") + .arg(catalog_name) + .assert() + .stdout(IsJson) + .success(); +} From 038a3132068903f50e60598afc428bdc0fb86ec4 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 15:25:59 -0400 Subject: [PATCH 07/15] input format support --- src/cmds/tables.rs | 22 ++++++++++++---------- src/error.rs | 4 ++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/cmds/tables.rs b/src/cmds/tables.rs index 173178f..86a7cf1 100644 --- a/src/cmds/tables.rs +++ b/src/cmds/tables.rs @@ -1,10 +1,10 @@ +use crate::cli::Format; use crate::cli::MetastoreClient; use crate::cli::RunCommand; use crate::error::CliError; -use clap::{Args, ValueEnum}; +use clap::Args; use nektar::{TThriftHiveMetastoreSyncClient, Table}; use std::convert::TryInto; -use std::io::prelude::*; use std::path::PathBuf; use std::{ fs::File, @@ -23,14 +23,12 @@ impl RunCommand> for GetTable { } } -#[derive(ValueEnum, Debug, Clone)] -pub enum TableType { - External, - Internal, -} - #[derive(Debug, Args)] pub struct CreateTable { + /// The input format for the table definition file + #[arg(value_enum, short='f', long="format", default_value_t = Format::Json)] + format: Format, + /// A file path for table definition table_definition_file: PathBuf, } @@ -43,8 +41,12 @@ impl RunCommand
for CreateTable { .as_secs() .try_into() .unwrap(); - let file = File::open(self.table_definition_file).unwrap(); - let mut table: Table = serde_json::from_reader(file).unwrap(); + let file = File::open(self.table_definition_file)?; + let mut table: Table = match self.format { + Format::Json => serde_json::from_reader(file).unwrap(), + #[cfg(feature = "yaml")] + Format::Yaml => serde_yaml::from_reader(file).unwrap(), + }; table.create_time = Some(since_epoch); Ok(client.create_table(table.clone())?).map(|_| table) } diff --git a/src/error.rs b/src/error.rs index 4e16b14..77f22ae 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,4 +1,5 @@ use serde::{ser::SerializeStruct, Serialize}; +use std::io::Error as IoError; use thiserror::Error; #[derive(Error, Debug)] @@ -11,6 +12,8 @@ pub enum CliError { JsonSerdeError(#[from] serde_json::Error), #[error("Could not serialize to json")] YamlSerdeError(#[from] serde_yaml::Error), + #[error(transparent)] + IoError(#[from] IoError), } impl Serialize for CliError { @@ -27,6 +30,7 @@ impl Serialize for CliError { } CliError::JsonSerdeError(e) => serializer.serialize_str(&e.to_string()), CliError::YamlSerdeError(e) => serializer.serialize_str(&e.to_string()), + CliError::IoError(e) => serializer.serialize_str(&e.to_string()), } } } From 9c36fe178319c267d9a0cee99aa48462ed79b9eb Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 15:36:18 -0400 Subject: [PATCH 08/15] implement drop table --- Cargo.lock | 62 +++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/cli.rs | 3 +++ src/cmds/tables.rs | 24 ++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 2769873..eb18cfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.4.0" @@ -345,6 +351,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "tempfile", + "thiserror", + "zeroize", +] + [[package]] name = "difflib" version = "0.4.0" @@ -420,6 +439,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "float-cmp" version = "0.9.0" @@ -718,6 +743,7 @@ dependencies = [ "clap", "console", "ctor", + "dialoguer", "env_logger", "lazy_static", "log", @@ -888,6 +914,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.9.5" @@ -929,7 +964,7 @@ version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ - "bitflags", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", @@ -1019,6 +1054,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "slab" version = "0.4.9" @@ -1062,6 +1103,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "termcolor" version = "1.3.0" @@ -1443,3 +1497,9 @@ name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/Cargo.toml b/Cargo.toml index 734ceb7..66a9b82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ serde = { version = "1.0.174", features = ["derive"] } serde_json = "1.0.103" serde_yaml = { version = "0.9.25", optional = true } assert_cmd = "2.0.12" +dialoguer = "0.11.0" [dev-dependencies] async_once = "0.2.6" diff --git a/src/cli.rs b/src/cli.rs index 06f7208..4588c7d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,6 +9,7 @@ use thrift::transport::{ }; use crate::cmds::catalogs::GetCatalogs; +use crate::cmds::tables::DropTable; use crate::cmds::{ catalogs::{CreateCatalog, GetCatalog}, databases::GetDatabases, @@ -55,6 +56,7 @@ pub enum Commands { GetDatabases(GetDatabases), CreateCatalog(CreateCatalog), CreateTable(CreateTable), + DropTable(DropTable), } fn serialize(f: Format, v: T) -> Result { @@ -96,6 +98,7 @@ impl Cli { Commands::CreateTable(create_table) => { serialize(self.format, create_table.run(client)?) } + Commands::DropTable(drop_table) => serialize(self.format, drop_table.run(client)?), } } } diff --git a/src/cmds/tables.rs b/src/cmds/tables.rs index 86a7cf1..4b46c57 100644 --- a/src/cmds/tables.rs +++ b/src/cmds/tables.rs @@ -3,6 +3,7 @@ use crate::cli::MetastoreClient; use crate::cli::RunCommand; use crate::error::CliError; use clap::Args; +use dialoguer::Confirm; use nektar::{TThriftHiveMetastoreSyncClient, Table}; use std::convert::TryInto; use std::path::PathBuf; @@ -51,3 +52,26 @@ impl RunCommand
for CreateTable { Ok(client.create_table(table.clone())?).map(|_| table) } } + +#[derive(Debug, Args)] +pub struct DropTable { + db_name: String, + table_name: String, + #[arg(long = "delete")] + delete: bool, +} + +impl RunCommand<()> for DropTable { + fn run(self, mut client: MetastoreClient) -> Result<(), CliError> { + let confirmation = Confirm::new() + .with_prompt(format!("Delete {}?", self.table_name)) + .interact() + .unwrap(); + + if confirmation { + Ok(client.drop_table(self.db_name, self.table_name, self.delete)?) + } else { + Ok(()) + } + } +} From b9b0abcc42c4a37e91bee8030b3f2189c0fdd017 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 6 Oct 2023 15:39:30 -0400 Subject: [PATCH 09/15] update docstrings, readme --- README.md | 16 ++++++++++------ src/cmds/databases.rs | 1 + src/cmds/partitions.rs | 14 ++++++++------ src/cmds/tables.rs | 3 +++ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 742361e..6ec3443 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,18 @@ Install via `cargo install` or download a binary directly from releases. ``` A fast, lightweight CLI for Hive Metastore -Usage: nektar [OPTIONS] +Usage: nektar [OPTIONS] Commands: - get-table - get-catalog - get-partitions - get-partition-names-by-parts - get-databases + get-table Get a single table by database and table name + get-catalog Get a single catalog by name + get-catalogs Get a list of all catalogs in metastore + get-partitions Get the partitions of a table + get-partition-names-by-parts Get the partitions of a table by partition value + get-databases Get all databases in the metastore + create-catalog Create a catalog + create-table Create a table from a table definition file + drop-table Drop a single table by database and table name help Print this message or the help of the given subcommand(s) Arguments: diff --git a/src/cmds/databases.rs b/src/cmds/databases.rs index 0780925..8f42ca8 100644 --- a/src/cmds/databases.rs +++ b/src/cmds/databases.rs @@ -4,6 +4,7 @@ use crate::error::CliError; use clap::Args; use nektar::TThriftHiveMetastoreSyncClient; +/// Get all databases in the metastore #[derive(Debug, Args)] pub struct GetDatabases; diff --git a/src/cmds/partitions.rs b/src/cmds/partitions.rs index 0de5a61..e291e91 100644 --- a/src/cmds/partitions.rs +++ b/src/cmds/partitions.rs @@ -4,12 +4,20 @@ use crate::error::CliError; use clap::Args; use nektar::TThriftHiveMetastoreSyncClient; +/// Get the partitions of a table #[derive(Debug, Args)] pub struct GetPartitions { database: String, table: String, } +impl RunCommand> for GetPartitions { + fn run(self, mut client: MetastoreClient) -> Result, CliError> { + Ok(client.get_partition_names(self.database, self.table, 10)?) + } +} + +/// Get the partitions of a table by partition value #[derive(Debug, Args)] pub struct GetPartitionNamesByParts { database: String, @@ -19,12 +27,6 @@ pub struct GetPartitionNamesByParts { max_parts: i16, } -impl RunCommand> for GetPartitions { - fn run(self, mut client: MetastoreClient) -> Result, CliError> { - Ok(client.get_partition_names(self.database, self.table, 10)?) - } -} - impl RunCommand> for GetPartitionNamesByParts { fn run(self, mut client: MetastoreClient) -> Result, CliError> { Ok(client.get_partition_names_ps( diff --git a/src/cmds/tables.rs b/src/cmds/tables.rs index 4b46c57..5e5124f 100644 --- a/src/cmds/tables.rs +++ b/src/cmds/tables.rs @@ -12,6 +12,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +/// Get a single table by database and table name #[derive(Debug, Args)] pub struct GetTable { database: String, @@ -24,6 +25,7 @@ impl RunCommand> for GetTable { } } +/// Create a table from a table definition file #[derive(Debug, Args)] pub struct CreateTable { /// The input format for the table definition file @@ -53,6 +55,7 @@ impl RunCommand
for CreateTable { } } +/// Drop a single table by database and table name #[derive(Debug, Args)] pub struct DropTable { db_name: String, From 884fe5c236c7829c9e72be179870c88c78bff746 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Tue, 10 Oct 2023 15:37:54 -0400 Subject: [PATCH 10/15] add metastore startup script --- metastore.sh | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 metastore.sh diff --git a/metastore.sh b/metastore.sh new file mode 100755 index 0000000..8070810 --- /dev/null +++ b/metastore.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +readonly DOCKER="docker" +readonly HIVE_IMAGE_TAG="apache/hive:4.0.0-beta-2-SNAPSHOT" +readonly CONTAINER_NAME="metastore-standalone" +readonly MODE="${1:-}" + +container_exists() { + local id + id=$("${DOCKER}" container ls --all --quiet --filter "name=${CONTAINER_NAME}") + if [ "$id" != "" ] + then + return 0 + fi + + return 1 +} + +usage() { + echo "" + echo "convenience script to run trino locally for testing" + echo "" + echo "usage: $0 start/stop/rm" +} + +if [ -z "${MODE}" ]; then + usage + exit 0 +fi + +case "$MODE" in +"start") + if container_exists + then + echo "Starting a previous container." + ${DOCKER} start "${CONTAINER_NAME}" + else + ${DOCKER} run --name "${CONTAINER_NAME}" --detach --publish 9083:9083 \ + --env SERVICE_OPTS="-Dhive.root.logger=console" \ + --env SERVICE_NAME=metastore \ + ${HIVE_IMAGE_TAG} + fi + ;; +"stop") + ${DOCKER} kill "${CONTAINER_NAME}" + ;; +"rm") + ${DOCKER} rm "${CONTAINER_NAME}" + ;; +*) + usage + ;; +esac From caa32617cab3a21e4f0acb7c6ff7a46d659a0903 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Tue, 10 Oct 2023 15:46:27 -0400 Subject: [PATCH 11/15] single test suite for now --- tests/integration_test.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index a19aeff..1ec7e8d 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -151,29 +151,3 @@ async fn test_create_and_get_catalog() { .stdout(IsJson) .success(); } - -#[tokio::test] -async fn test_create_table() { - METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); - let mut create_cmd = Command::cargo_bin("nektar").unwrap(); - let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); - let catalog_name = "test"; - create_cmd - .arg(format!("{}:{}", METASTORE_HOST, open_port)) - .arg("create-catalog") - .arg(catalog_name) - .arg("file:/opt/hive/data/warehouse") - .arg("-d") - .arg("a description") - .assert() - .success(); - - let mut get_cmd = Command::cargo_bin("nektar").unwrap(); - get_cmd - .arg(format!("{}:{}", METASTORE_HOST, open_port)) - .arg("get-catalog") - .arg(catalog_name) - .assert() - .stdout(IsJson) - .success(); -} From e8ea76600e88d419f95b590875b2160583cbc473 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Fri, 13 Oct 2023 15:11:58 -0400 Subject: [PATCH 12/15] use bridge network on github actions --- .github/workflows/build.yml | 1 + .github/workflows/release.yml | 1 + tests/integration_test.rs | 35 +++++++++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16763b4..a1558ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,7 @@ on: [pull_request] env: CARGO_TERM_COLOR: always + GITHUB_ACTIONS: true jobs: build-linux-x86_64: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 34f7363..51ad7fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ on: env: CARGO_TERM_COLOR: always + GITHUB_ACTIONS: true jobs: build-linux-x86_64: diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 1ec7e8d..44878af 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -5,7 +5,7 @@ use predicates::json::IsJson; use assert_cmd::Command; use ctor::{ctor, dtor}; use lazy_static::lazy_static; -use std::{future::Future, thread}; +use std::{env, fmt::Display, future::Future, net::IpAddr, thread}; use testcontainers::clients::Cli; use testcontainers::core::WaitFor; use testcontainers::images::generic::GenericImage; @@ -48,9 +48,20 @@ enum ContainerCommands { Stop, } +struct HostPort { + host: IpAddr, + port: u16, +} + +impl Display for HostPort { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.host, self.port) + } +} + lazy_static! { static ref METASTORE_IN: Channel = channel(); - static ref METASTORE_HOST_PORT: Channel = channel(); + static ref METASTORE_HOST_PORT: Channel = channel(); static ref METASTORE_STOPPED: Channel<()> = channel(); } @@ -67,6 +78,10 @@ async fn set_up() { .with_env_var(SERVICE_NAME_VAR, SERVICE_NAME) .with_env_var(SERVICE_OPTS_VAR, SERVICE_OPTS); let hive_metastore = docker.run(hive_image); + let host = match env::var("GITHUB_ACTIONS") { + Ok(_) => hive_metastore.get_bridge_ip_address(), + Err(_) => "127.0.0.1".parse().unwrap(), + }; let open_port = hive_metastore.ports().map_to_host_port_ipv4(METASTORE_PORT); log::info!("{}:{}", METASTORE_HOST, METASTORE_PORT); @@ -74,9 +89,13 @@ async fn set_up() { while let Some(command) = rx.recv().await { println!("Received container command: {:?}", command); match command { - ContainerCommands::FetchPort => { - METASTORE_HOST_PORT.tx.send(open_port.unwrap()).unwrap() - } + ContainerCommands::FetchPort => METASTORE_HOST_PORT + .tx + .send(HostPort { + host: host, + port: open_port.unwrap(), + }) + .unwrap(), ContainerCommands::Stop => { hive_metastore.stop(); METASTORE_STOPPED.tx.send(()).unwrap(); @@ -119,7 +138,7 @@ async fn test_get_default_catalog() { METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); let mut cmd = Command::cargo_bin("nektar").unwrap(); let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); - cmd.arg(format!("{}:{}", METASTORE_HOST, open_port)) + cmd.arg(format!("{}", open_port)) .arg("get-catalog") .arg("hive") .assert() @@ -133,7 +152,7 @@ async fn test_create_and_get_catalog() { let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); let catalog_name = "test"; create_cmd - .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg(format!("{}", open_port)) .arg("create-catalog") .arg(catalog_name) .arg("file:/opt/hive/data/warehouse") @@ -144,7 +163,7 @@ async fn test_create_and_get_catalog() { let mut get_cmd = Command::cargo_bin("nektar").unwrap(); get_cmd - .arg(format!("{}:{}", METASTORE_HOST, open_port)) + .arg(format!("{}", open_port)) .arg("get-catalog") .arg(catalog_name) .assert() From 1b22e706a5da880c38ce1c0893ddd624148ef2cf Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Mon, 16 Oct 2023 15:27:20 -0400 Subject: [PATCH 13/15] try host network --- .github/workflows/build.yml | 32 ++- Cargo.lock | 475 ++++++++++++++++++++++++++---------- Cargo.toml | 3 +- tests/integration_test.rs | 90 ++++--- 4 files changed, 424 insertions(+), 176 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1558ce..c90786e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,15 +8,29 @@ env: jobs: build-linux-x86_64: runs-on: ubuntu-latest + services: + metastore: + image: apache/hive:4.0.0-beta-2-SNAPSHOT + env: + SERVICE_NAME: metastore + SERVICE_OPTS: "-Dhive.root.logger=console" + ports: + - 9083:9083 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install target + uses: actions-rs/toolchain@v1 + with: + toolchain: stable - name: Build run: cargo build --verbose --release - name: Test - run: cargo test --verbose - - name: Archive binaries + run: cargo test -- --show-output + env: + RUST_LOG: trace + - name: Archive binaries uses: actions/upload-artifact@v3 with: name: nektar-x86_64 @@ -24,24 +38,22 @@ jobs: target/release/nektar build-aarch64-darwin: - runs-on: macos-latest + runs-on: macos-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install aarch64 target + - name: Install aarch64 target uses: actions-rs/toolchain@v1 with: - target: aarch64-apple-darwin + target: aarch64-apple-darwin toolchain: stable - name: Build run: cargo build --verbose --release --target=aarch64-apple-darwin - - name: Test - run: cargo test --verbose - - name: Archive binaries + - name: Archive binaries uses: actions/upload-artifact@v3 with: - name: nektar-aarch64-apple-darwin + name: nektar-aarch64-apple-darwin path: | - target/aarch64-apple-darwin/release/nektar + target/aarch64-apple-darwin/release/nektar diff --git a/Cargo.lock b/Cargo.lock index eb18cfb..fe653da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,21 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "0.5.0" @@ -104,6 +89,17 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "async_once" version = "0.2.6" @@ -131,6 +127,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -152,13 +154,40 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "url", + "winapi", +] + [[package]] name = "bollard-stubs" -version = "1.41.0" +version = "1.42.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" dependencies = [ - "chrono", "serde", "serde_with", ] @@ -174,18 +203,18 @@ dependencies = [ "serde", ] -[[package]] -name = "bumpalo" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" - [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + [[package]] name = "cc" version = "1.0.83" @@ -207,21 +236,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-targets 0.48.5", -] - [[package]] name = "clap" version = "4.4.4" @@ -281,12 +295,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - [[package]] name = "cpufeatures" version = "0.2.9" @@ -460,6 +468,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.28" @@ -576,6 +593,31 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "h2" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.0" @@ -609,6 +651,40 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" @@ -616,26 +692,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] -name = "iana-time-zone" -version = "0.1.57" +name = "hyper" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "hyperlocal" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" dependencies = [ - "cc", + "futures-util", + "hex", + "hyper", + "pin-project", + "tokio", ] [[package]] @@ -644,6 +734,26 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.0.0" @@ -651,7 +761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.0", ] [[package]] @@ -686,15 +796,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" -[[package]] -name = "js-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" -dependencies = [ - "wasm-bindgen", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -734,6 +835,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "nektar" version = "0.0.2" @@ -817,6 +929,32 @@ dependencies = [ "num-traits", ] +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1008,6 +1146,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "1.14.0" @@ -1036,7 +1186,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -1069,6 +1219,26 @@ dependencies = [ "autocfg", ] +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1133,10 +1303,12 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "testcontainers" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e2b1567ca8a2b819ea7b28c92be35d9f76fb9edb214321dcc86eb96023d1f87" +checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00" dependencies = [ + "async-trait", + "bollard", "bollard-stubs", "futures", "hex", @@ -1146,6 +1318,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tokio", ] [[package]] @@ -1190,6 +1363,21 @@ dependencies = [ "threadpool", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.32.0" @@ -1197,8 +1385,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", "pin-project-lite", + "socket2 0.5.4", "tokio-macros", + "windows-sys 0.48.0", ] [[package]] @@ -1212,6 +1406,51 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "tokio-util" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "try_from" version = "0.3.2" @@ -1227,12 +1466,27 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.11" @@ -1245,6 +1499,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1267,64 +1532,19 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" -dependencies = [ - "cfg-if 1.0.0", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.87" +name = "want" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-backend", - "wasm-bindgen-shared", + "try-lock", ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.87" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "winapi" @@ -1357,15 +1577,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index 66a9b82..051cdc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,10 @@ dialoguer = "0.11.0" [dev-dependencies] async_once = "0.2.6" ctor = "0.2.5" +env_logger = "0.10.0" lazy_static = "1.4.0" predicates = "3.0.4" -testcontainers = "0.14.0" +testcontainers = { version = "0.15.0", features = ["experimental"] } tokio = { version = "1.32.0", features = ["macros", "sync", "rt"] } [features] diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 44878af..e1b2fa6 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,4 +1,5 @@ mod predicates; +use log::*; use predicates::json::IsJson; @@ -6,9 +7,9 @@ use assert_cmd::Command; use ctor::{ctor, dtor}; use lazy_static::lazy_static; use std::{env, fmt::Display, future::Future, net::IpAddr, thread}; -use testcontainers::clients::Cli; use testcontainers::core::WaitFor; -use testcontainers::images::generic::GenericImage; +use testcontainers::GenericImage; +use testcontainers::{clients::Cli, RunnableImage}; use tokio::{ runtime, sync::{ @@ -49,7 +50,7 @@ enum ContainerCommands { } struct HostPort { - host: IpAddr, + host: String, port: u16, } @@ -70,36 +71,58 @@ lazy_static! { /// Necessary to support starting one long lived testcontainer /// for all test methods. async fn set_up() { - let docker = Cli::default(); - let msg_await = WaitFor::message_on_stderr(READY_MSG.to_string()); - let hive_image = GenericImage::new("apache/hive", HIVE_VERSION) - .with_wait_for(msg_await) - .with_exposed_port(METASTORE_PORT) - .with_env_var(SERVICE_NAME_VAR, SERVICE_NAME) - .with_env_var(SERVICE_OPTS_VAR, SERVICE_OPTS); - let hive_metastore = docker.run(hive_image); - let host = match env::var("GITHUB_ACTIONS") { - Ok(_) => hive_metastore.get_bridge_ip_address(), - Err(_) => "127.0.0.1".parse().unwrap(), - }; - let open_port = hive_metastore.ports().map_to_host_port_ipv4(METASTORE_PORT); - - log::info!("{}:{}", METASTORE_HOST, METASTORE_PORT); - let mut rx = METASTORE_IN.rx.lock().await; - while let Some(command) = rx.recv().await { - println!("Received container command: {:?}", command); - match command { - ContainerCommands::FetchPort => METASTORE_HOST_PORT - .tx - .send(HostPort { - host: host, - port: open_port.unwrap(), - }) - .unwrap(), - ContainerCommands::Stop => { - hive_metastore.stop(); - METASTORE_STOPPED.tx.send(()).unwrap(); - rx.close(); + match env::var("GITHUB_ACTIONS") { + Err(_) => { + let docker = Cli::default(); + let msg_await = WaitFor::message_on_stderr(READY_MSG.to_string()); + let hive_image = RunnableImage::from( + GenericImage::new("apache/hive", HIVE_VERSION) + .with_wait_for(msg_await) + .with_exposed_port(METASTORE_PORT) + .with_env_var(SERVICE_NAME_VAR, SERVICE_NAME) + .with_env_var(SERVICE_OPTS_VAR, SERVICE_OPTS), + ) + .with_mapped_port((METASTORE_PORT, METASTORE_PORT)); + let hive_metastore = docker.run(hive_image); + let open_port = hive_metastore.ports().map_to_host_port_ipv4(METASTORE_PORT); + + log::info!("{}:{}", METASTORE_HOST, METASTORE_PORT); + let mut rx = METASTORE_IN.rx.lock().await; + while let Some(command) = rx.recv().await { + println!("Received container command: {:?}", command); + match command { + ContainerCommands::FetchPort => METASTORE_HOST_PORT + .tx + .send(HostPort { + host: "localhost".to_string(), + port: open_port.unwrap(), + }) + .unwrap(), + ContainerCommands::Stop => { + hive_metastore.stop(); + METASTORE_STOPPED.tx.send(()).unwrap(); + rx.close(); + } + } + } + } + Ok(_) => { + let mut rx = METASTORE_IN.rx.lock().await; + while let Some(command) = rx.recv().await { + println!("Received container command: {:?}", command); + match command { + ContainerCommands::FetchPort => METASTORE_HOST_PORT + .tx + .send(HostPort { + host: "localhost".to_string(), + port: 9083, + }) + .unwrap(), + ContainerCommands::Stop => { + METASTORE_STOPPED.tx.send(()).unwrap(); + rx.close(); + } + } } } } @@ -135,6 +158,7 @@ async fn clean_up() { #[tokio::test] async fn test_get_default_catalog() { + env_logger::builder().is_test(true).try_init(); METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); let mut cmd = Command::cargo_bin("nektar").unwrap(); let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); From 75148c54b95b44047b20dc4440d8e05cdf1e108b Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Tue, 17 Oct 2023 09:29:20 -0400 Subject: [PATCH 14/15] add wait logic to metastore script --- metastore.sh | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/metastore.sh b/metastore.sh index 8070810..c671288 100755 --- a/metastore.sh +++ b/metastore.sh @@ -19,11 +19,36 @@ container_exists() { return 1 } +err_retry() { + local exit_code=$1 + local attempts=$2 + local sleep_millis=$3 + shift 3 + for attempt in `seq 1 $attempts`; do + echo "Attempt $attempt of $attempts" + # This weird construction lets us capture return codes under -o errexit + "$@" && local rc=$? || local rc=$? + if [[ ! $rc -eq $exit_code ]]; then + return $rc + fi + if [[ $attempt -eq $attempts ]]; then + return $rc + fi + local sleep_s="$((($attempt * $attempt * $sleep_millis) / 1000))" + sleep $sleep_s + done +} + +metastore_available() { + (nc -z localhost 9083) + return $? +} + usage() { echo "" - echo "convenience script to run trino locally for testing" + echo "convenience script to run hive metastore locally for testing" echo "" - echo "usage: $0 start/stop/rm" + echo "usage: $0 start | await | stop | rm " } if [ -z "${MODE}" ]; then @@ -44,6 +69,11 @@ case "$MODE" in ${HIVE_IMAGE_TAG} fi ;; +"await") + echo "Waiting for metastore availability..." + err_retry 1 5 1000 metastore_available + echo "metastore available!" + ;; "stop") ${DOCKER} kill "${CONTAINER_NAME}" ;; From 8375181907213b45940e64e46b3adb6f909b04a8 Mon Sep 17 00:00:00 2001 From: Mark Grey Date: Tue, 17 Oct 2023 12:23:35 -0400 Subject: [PATCH 15/15] abandon testcontainers approach for now --- .github/workflows/build.yml | 3 + .github/workflows/release.yml | 21 +- Cargo.lock | 831 +--------------------------------- Cargo.toml | 5 - tests/integration_test.rs | 224 ++------- 5 files changed, 64 insertions(+), 1020 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c90786e..6f8bb27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,10 +26,13 @@ jobs: toolchain: stable - name: Build run: cargo build --verbose --release + - name: Await metastore container + run: ./metastore.sh await - name: Test run: cargo test -- --show-output env: RUST_LOG: trace + RUSTFLAGS: '--cfg ci' - name: Archive binaries uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51ad7fd..8e4014f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,14 +14,31 @@ env: jobs: build-linux-x86_64: runs-on: ubuntu-latest + services: + metastore: + image: apache/hive:4.0.0-beta-2-SNAPSHOT + env: + SERVICE_NAME: metastore + SERVICE_OPTS: "-Dhive.root.logger=console" + ports: + - 9083:9083 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install target + uses: actions-rs/toolchain@v1 + with: + toolchain: stable - name: Build run: cargo build --verbose --release + - name: Await metastore container + run: ./metastore.sh await - name: Test - run: cargo test --verbose + run: cargo test -- --show-output + env: + RUST_LOG: trace + RUSTFLAGS: '--cfg ci' - name: Archive binaries uses: actions/upload-artifact@v3 with: @@ -43,8 +60,6 @@ jobs: toolchain: stable - name: Build run: cargo build --verbose --release --target=aarch64-apple-darwin - - name: Test - run: cargo test --verbose - name: Archive binaries uses: actions/upload-artifact@v3 with: diff --git a/Cargo.lock b/Cargo.lock index fe653da..11860c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "aho-corasick" version = "1.1.1" @@ -89,50 +74,12 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "async-trait" -version = "0.1.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "async_once" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82" - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "bitflags" version = "1.3.2" @@ -145,53 +92,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bollard" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82e7850583ead5f8bbef247e2a3c37a19bd576e8420cd262a6711921827e1e5" -dependencies = [ - "base64", - "bollard-stubs", - "bytes", - "futures-core", - "futures-util", - "hex", - "http", - "hyper", - "hyperlocal", - "log", - "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util", - "url", - "winapi", -] - -[[package]] -name = "bollard-stubs" -version = "1.42.0-rc.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" -dependencies = [ - "serde", - "serde_with", -] - [[package]] name = "bstr" version = "1.6.2" @@ -209,12 +109,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" - [[package]] name = "cc" version = "1.0.83" @@ -267,7 +161,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn", ] [[package]] @@ -295,70 +189,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "cpufeatures" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "ctor" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" -dependencies = [ - "quote", - "syn 2.0.37", -] - -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core", - "quote", - "syn 1.0.109", -] - [[package]] name = "dialoguer" version = "0.11.0" @@ -378,17 +208,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - [[package]] name = "doc-comment" version = "0.3.3" @@ -462,162 +281,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" - -[[package]] -name = "h2" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 1.9.3", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -636,124 +299,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.9", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyperlocal" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" -dependencies = [ - "futures-util", - "hex", - "hyper", - "pin-project", - "tokio", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -761,7 +312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -826,48 +377,23 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - [[package]] name = "nektar" version = "0.0.2" dependencies = [ "assert_cmd", - "async_once", "clap", "console", - "ctor", "dialoguer", "env_logger", - "lazy_static", "log", "ordered-float 3.9.1", "predicates", "serde", "serde_json", "serde_yaml", - "testcontainers", "thiserror", "thrift", - "tokio", "try_from", ] @@ -896,21 +422,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - [[package]] name = "ordered-float" version = "2.10.0" @@ -929,50 +440,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - [[package]] name = "predicates" version = "3.0.4" @@ -1022,36 +489,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -1090,12 +527,6 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - [[package]] name = "rustix" version = "0.38.14" @@ -1132,7 +563,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn", ] [[package]] @@ -1146,122 +577,31 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" -dependencies = [ - "serde", - "serde_with_macros", -] - -[[package]] -name = "serde_with_macros" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "serde_yaml" version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.0", + "indexmap", "itoa", "ryu", "serde", "unsafe-libyaml", ] -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - [[package]] name = "shell-words" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.37" @@ -1301,26 +641,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" -[[package]] -name = "testcontainers" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00" -dependencies = [ - "async-trait", - "bollard", - "bollard-stubs", - "futures", - "hex", - "hmac", - "log", - "rand", - "serde", - "serde_json", - "sha2", - "tokio", -] - [[package]] name = "thiserror" version = "1.0.48" @@ -1338,7 +658,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn", ] [[package]] @@ -1363,94 +683,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "socket2 0.5.4", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "tokio-util" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - [[package]] name = "try_from" version = "0.3.2" @@ -1460,33 +692,12 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-width" version = "0.1.11" @@ -1499,29 +710,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" -[[package]] -name = "url" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "wait-timeout" version = "0.2.0" @@ -1531,21 +725,6 @@ dependencies = [ "libc", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 051cdc2..2ce5736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,13 +25,8 @@ assert_cmd = "2.0.12" dialoguer = "0.11.0" [dev-dependencies] -async_once = "0.2.6" -ctor = "0.2.5" env_logger = "0.10.0" -lazy_static = "1.4.0" predicates = "3.0.4" -testcontainers = { version = "0.15.0", features = ["experimental"] } -tokio = { version = "1.32.0", features = ["macros", "sync", "rt"] } [features] default = ["yaml"] diff --git a/tests/integration_test.rs b/tests/integration_test.rs index e1b2fa6..3e92c15 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,196 +1,48 @@ mod predicates; -use log::*; -use predicates::json::IsJson; +#[cfg(ci)] +mod tests { -use assert_cmd::Command; -use ctor::{ctor, dtor}; -use lazy_static::lazy_static; -use std::{env, fmt::Display, future::Future, net::IpAddr, thread}; -use testcontainers::core::WaitFor; -use testcontainers::GenericImage; -use testcontainers::{clients::Cli, RunnableImage}; -use tokio::{ - runtime, - sync::{ - mpsc::{self, UnboundedReceiver, UnboundedSender}, - Mutex, - }, -}; + use crate::predicates::json::IsJson; -const METASTORE_HOST: &str = "localhost"; -const METASTORE_PORT: u16 = 9083; -const HIVE_VERSION: &str = "4.0.0-beta-2-SNAPSHOT"; -const SERVICE_NAME_VAR: &str = "SERVICE_NAME"; -const SERVICE_NAME: &str = "metastore"; -const SERVICE_OPTS_VAR: &str = "SERVICE_OPTS"; -const SERVICE_OPTS: &str = "-Dhive.root.logger=console"; -const READY_MSG: &str = "Successfully created a default database with name: default"; + use assert_cmd::Command; -/// Channels for communication with container started on thread -struct Channel { - tx: UnboundedSender, - rx: Mutex>, -} + const METASTORE_HOST: &str = "localhost"; + const METASTORE_PORT: u16 = 9083; -/// Channel constructor -fn channel() -> Channel { - let (tx, rx) = mpsc::unbounded_channel(); - Channel { - tx, - rx: Mutex::new(rx), + #[test] + fn test_get_default_catalog() { + let _ = env_logger::builder().is_test(true).try_init(); + let mut cmd = Command::cargo_bin("nektar").unwrap(); + cmd.arg(format!("{}:{}", METASTORE_HOST, METASTORE_PORT)) + .arg("get-catalog") + .arg("hive") + .assert() + .success(); } -} - -/// Messages for communicating with container -#[derive(Debug)] -enum ContainerCommands { - FetchPort, - Stop, -} -struct HostPort { - host: String, - port: u16, -} - -impl Display for HostPort { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}:{}", self.host, self.port) + #[test] + fn test_create_and_get_catalog() { + let _ = env_logger::builder().is_test(true).try_init(); + let mut create_cmd = Command::cargo_bin("nektar").unwrap(); + let catalog_name = "test"; + create_cmd + .arg(format!("{}:{}", METASTORE_HOST, METASTORE_PORT)) + .arg("create-catalog") + .arg(catalog_name) + .arg("file:/opt/hive/data/warehouse") + .arg("-d") + .arg("a description") + .assert() + .success(); + + let mut get_cmd = Command::cargo_bin("nektar").unwrap(); + get_cmd + .arg(format!("{}:{}", METASTORE_HOST, METASTORE_PORT)) + .arg("get-catalog") + .arg(catalog_name) + .assert() + .stdout(IsJson) + .success(); } } - -lazy_static! { - static ref METASTORE_IN: Channel = channel(); - static ref METASTORE_HOST_PORT: Channel = channel(); - static ref METASTORE_STOPPED: Channel<()> = channel(); -} - -/// Start testcontainers, open channels for communication -/// -/// Necessary to support starting one long lived testcontainer -/// for all test methods. -async fn set_up() { - match env::var("GITHUB_ACTIONS") { - Err(_) => { - let docker = Cli::default(); - let msg_await = WaitFor::message_on_stderr(READY_MSG.to_string()); - let hive_image = RunnableImage::from( - GenericImage::new("apache/hive", HIVE_VERSION) - .with_wait_for(msg_await) - .with_exposed_port(METASTORE_PORT) - .with_env_var(SERVICE_NAME_VAR, SERVICE_NAME) - .with_env_var(SERVICE_OPTS_VAR, SERVICE_OPTS), - ) - .with_mapped_port((METASTORE_PORT, METASTORE_PORT)); - let hive_metastore = docker.run(hive_image); - let open_port = hive_metastore.ports().map_to_host_port_ipv4(METASTORE_PORT); - - log::info!("{}:{}", METASTORE_HOST, METASTORE_PORT); - let mut rx = METASTORE_IN.rx.lock().await; - while let Some(command) = rx.recv().await { - println!("Received container command: {:?}", command); - match command { - ContainerCommands::FetchPort => METASTORE_HOST_PORT - .tx - .send(HostPort { - host: "localhost".to_string(), - port: open_port.unwrap(), - }) - .unwrap(), - ContainerCommands::Stop => { - hive_metastore.stop(); - METASTORE_STOPPED.tx.send(()).unwrap(); - rx.close(); - } - } - } - } - Ok(_) => { - let mut rx = METASTORE_IN.rx.lock().await; - while let Some(command) = rx.recv().await { - println!("Received container command: {:?}", command); - match command { - ContainerCommands::FetchPort => METASTORE_HOST_PORT - .tx - .send(HostPort { - host: "localhost".to_string(), - port: 9083, - }) - .unwrap(), - ContainerCommands::Stop => { - METASTORE_STOPPED.tx.send(()).unwrap(); - rx.close(); - } - } - } - } - } -} - -/// Helper function for blocking run, used to start testcontainer -fn execute_blocking(f: F) { - runtime::Builder::new_current_thread() - .build() - .unwrap() - .block_on(f); -} - -#[ctor] -/// Start container on process start -fn on_startup() { - thread::spawn(|| execute_blocking(set_up())); -} - -#[dtor] -/// Kill container on process end -fn on_shutdown() { - execute_blocking(clean_up()); -} - -/// Destructor, kill container and notify channels -async fn clean_up() { - METASTORE_IN.tx.send(ContainerCommands::Stop).unwrap(); - METASTORE_STOPPED.rx.lock().await.recv().await; -} - -// BEGIN: test cases - -#[tokio::test] -async fn test_get_default_catalog() { - env_logger::builder().is_test(true).try_init(); - METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); - let mut cmd = Command::cargo_bin("nektar").unwrap(); - let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); - cmd.arg(format!("{}", open_port)) - .arg("get-catalog") - .arg("hive") - .assert() - .success(); -} - -#[tokio::test] -async fn test_create_and_get_catalog() { - METASTORE_IN.tx.send(ContainerCommands::FetchPort).unwrap(); - let mut create_cmd = Command::cargo_bin("nektar").unwrap(); - let open_port = METASTORE_HOST_PORT.rx.lock().await.recv().await.unwrap(); - let catalog_name = "test"; - create_cmd - .arg(format!("{}", open_port)) - .arg("create-catalog") - .arg(catalog_name) - .arg("file:/opt/hive/data/warehouse") - .arg("-d") - .arg("a description") - .assert() - .success(); - - let mut get_cmd = Command::cargo_bin("nektar").unwrap(); - get_cmd - .arg(format!("{}", open_port)) - .arg("get-catalog") - .arg(catalog_name) - .assert() - .stdout(IsJson) - .success(); -}