diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml new file mode 100644 index 0000000000000..3b5832779d6d3 --- /dev/null +++ b/.github/workflows/post-merge.yml @@ -0,0 +1,36 @@ +# Workflow that runs after a merge to master, analyses changes in test executions +# and posts the result to the merged PR. + +name: Post merge analysis + +on: + push: + branches: + - master + +jobs: + analysis: + runs-on: ubuntu-24.04 + if: github.repository == 'rust-lang/rust' + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + - name: Perform analysis and send PR + run: | + # Get closest bors merge commit + PARENT_COMMIT=`git rev-list --author='bors ' -n1 --first-parent HEAD^1` + + # Find PRs for current and parent commit + HEAD_PR=`gh pr list --search "${{ github.sha }}" --state merged --json number --jq '.[0].number'` + + echo "Parent: ${PARENT_COMMIT}" + echo "HEAD: ${{ github.sha }} (#${HEAD_PR})" + + cd src/ci/citool + + echo "Post-merge analysis result" > output.log + cargo run --release post-merge-analysis ${PARENT_COMMIT} ${{ github.sha }} >> output.log + cat output.log + + gh pr comment ${HEAD_PR} -F output.log diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs index eb306550fc4a6..b6daac32a4412 100644 --- a/src/build_helper/src/metrics.rs +++ b/src/build_helper/src/metrics.rs @@ -74,7 +74,7 @@ pub struct Test { pub outcome: TestOutcome, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] #[serde(tag = "outcome", rename_all = "snake_case")] pub enum TestOutcome { Passed, diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index e5be2d42472b7..868346e447619 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "anstream" version = "0.6.18" @@ -38,7 +44,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -49,7 +55,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -58,6 +64,12 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "build_helper" version = "0.1.0" @@ -66,6 +78,27 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "bytes" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" + +[[package]] +name = "cc" +version = "1.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "citool" version = "0.1.0" @@ -77,6 +110,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "ureq", ] [[package]] @@ -134,7 +168,74 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", ] [[package]] @@ -149,6 +250,42 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -161,6 +298,162 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -208,18 +501,57 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + +[[package]] +name = "log" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "once_cell" version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project" version = "1.1.9" @@ -240,6 +572,12 @@ dependencies = [ "syn", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" version = "1.0.93" @@ -258,6 +596,61 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ring" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.19" @@ -309,18 +702,42 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "smallvec" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.98" @@ -332,6 +749,58 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "unicode-ident" version = "1.0.16" @@ -344,12 +813,110 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -422,3 +989,88 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index 11172832cb818..72aacc7d0c259 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -9,6 +9,7 @@ clap = { version = "4.5", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +ureq = { version = "3", features = ["json"] } build_helper = { path = "../../build_helper" } diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index cef92e998da1a..4011e0200bf4a 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,3 +1,4 @@ +mod merge_report; mod metrics; use std::collections::BTreeMap; @@ -8,6 +9,7 @@ use anyhow::Context; use clap::Parser; use serde_yaml::Value; +use crate::merge_report::post_merge_report; use crate::metrics::postprocess_metrics; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); @@ -350,6 +352,13 @@ enum Args { /// Usually, this will be GITHUB_STEP_SUMMARY on CI. summary_path: PathBuf, }, + /// Generate a report of test execution changes between two rustc commits. + PostMergeReport { + /// Parent commit to use as a base of the comparison. + parent: String, + /// Current commit that will be compared to `parent`. + current: String, + }, } #[derive(clap::ValueEnum, Clone)] @@ -384,6 +393,9 @@ fn main() -> anyhow::Result<()> { Args::PostprocessMetrics { metrics_path, summary_path } => { postprocess_metrics(&metrics_path, &summary_path)?; } + Args::PostMergeReport { current: commit, parent } => { + post_merge_report(load_db(default_jobs_file)?, parent, commit)?; + } } Ok(()) diff --git a/src/ci/citool/src/merge_report.rs b/src/ci/citool/src/merge_report.rs new file mode 100644 index 0000000000000..5dd662280f0f3 --- /dev/null +++ b/src/ci/citool/src/merge_report.rs @@ -0,0 +1,257 @@ +use std::cmp::Reverse; +use std::collections::HashMap; + +use anyhow::Context; +use build_helper::metrics::{JsonRoot, TestOutcome}; + +use crate::JobDatabase; +use crate::metrics::get_test_suites; + +type Sha = String; +type JobName = String; + +/// Computes a post merge CI analysis report between the `parent` and `current` commits. +pub fn post_merge_report(job_db: JobDatabase, parent: Sha, current: Sha) -> anyhow::Result<()> { + let jobs = download_all_metrics(&job_db, &parent, ¤t)?; + let diffs = aggregate_test_diffs(&jobs)?; + report_test_changes(diffs); + + Ok(()) +} + +struct JobMetrics { + parent: Option, + current: JsonRoot, +} + +/// Download before/after metrics for all auto jobs in the job database. +fn download_all_metrics( + job_db: &JobDatabase, + parent: &str, + current: &str, +) -> anyhow::Result> { + let mut jobs = HashMap::default(); + + for job in &job_db.auto_jobs { + eprintln!("Downloading metrics of job {}", job.name); + let metrics_parent = match download_job_metrics(&job.name, parent) { + Ok(metrics) => Some(metrics), + Err(error) => { + eprintln!( + r#"Did not find metrics for job `{}` at `{}`: {error:?}. +Maybe it was newly added?"#, + job.name, parent + ); + None + } + }; + let metrics_current = download_job_metrics(&job.name, current)?; + jobs.insert( + job.name.clone(), + JobMetrics { parent: metrics_parent, current: metrics_current }, + ); + } + Ok(jobs) +} + +fn download_job_metrics(job_name: &str, sha: &str) -> anyhow::Result { + let url = get_metrics_url(job_name, sha); + let mut response = ureq::get(&url).call()?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Cannot fetch metrics from {url}: {}\n{}", + response.status(), + response.body_mut().read_to_string()? + )); + } + let data: JsonRoot = response + .body_mut() + .read_json() + .with_context(|| anyhow::anyhow!("cannot deserialize metrics from {url}"))?; + Ok(data) +} + +fn get_metrics_url(job_name: &str, sha: &str) -> String { + let suffix = if job_name.ends_with("-alt") { "-alt" } else { "" }; + format!("https://ci-artifacts.rust-lang.org/rustc-builds{suffix}/{sha}/metrics-{job_name}.json") +} + +fn aggregate_test_diffs( + jobs: &HashMap, +) -> anyhow::Result> { + let mut job_diffs = vec![]; + + // Aggregate test suites + for (name, metrics) in jobs { + if let Some(parent) = &metrics.parent { + let tests_parent = aggregate_tests(parent); + let tests_current = aggregate_tests(&metrics.current); + let test_diffs = calculate_test_diffs(tests_parent, tests_current); + if !test_diffs.is_empty() { + job_diffs.push((name.clone(), test_diffs)); + } + } + } + + // Aggregate jobs with the same diff, as often the same diff will appear in many jobs + let job_diffs: HashMap, Vec> = + job_diffs.into_iter().fold(HashMap::new(), |mut acc, (job, diffs)| { + acc.entry(diffs).or_default().push(job); + acc + }); + + Ok(job_diffs + .into_iter() + .map(|(test_diffs, jobs)| AggregatedTestDiffs { jobs, test_diffs }) + .collect()) +} + +fn calculate_test_diffs( + reference: TestSuiteData, + current: TestSuiteData, +) -> Vec<(Test, TestOutcomeDiff)> { + let mut diffs = vec![]; + for (test, outcome) in ¤t.tests { + match reference.tests.get(test) { + Some(before) => { + if before != outcome { + diffs.push(( + test.clone(), + TestOutcomeDiff::ChangeOutcome { + before: before.clone(), + after: outcome.clone(), + }, + )); + } + } + None => diffs.push((test.clone(), TestOutcomeDiff::Added(outcome.clone()))), + } + } + for (test, outcome) in &reference.tests { + if !current.tests.contains_key(test) { + diffs.push((test.clone(), TestOutcomeDiff::Missing { before: outcome.clone() })); + } + } + + diffs +} + +/// Represents a difference in the outcome of tests between a base and a current commit. +#[derive(Debug)] +struct AggregatedTestDiffs { + /// All jobs that had the exact same test diffs. + jobs: Vec, + test_diffs: Vec<(Test, TestOutcomeDiff)>, +} + +#[derive(Eq, PartialEq, Hash, Debug)] +enum TestOutcomeDiff { + ChangeOutcome { before: TestOutcome, after: TestOutcome }, + Missing { before: TestOutcome }, + Added(TestOutcome), +} + +/// Aggregates test suite executions from all bootstrap invocations in a given CI job. +#[derive(Default)] +struct TestSuiteData { + tests: HashMap, +} + +#[derive(Hash, PartialEq, Eq, Debug, Clone)] +struct Test { + name: String, +} + +/// Extracts all tests from the passed metrics and map them to their outcomes. +fn aggregate_tests(metrics: &JsonRoot) -> TestSuiteData { + let mut tests = HashMap::new(); + let test_suites = get_test_suites(&metrics); + for suite in test_suites { + for test in &suite.tests { + let test_entry = Test { name: normalize_test_name(&test.name) }; + tests.insert(test_entry, test.outcome.clone()); + } + } + TestSuiteData { tests } +} + +/// Normalizes Windows-style path delimiters to Unix-style paths. +fn normalize_test_name(name: &str) -> String { + name.replace('\\', "/") +} + +/// Prints test changes in Markdown format to stdout. +fn report_test_changes(mut diffs: Vec) { + println!("## Test differences"); + if diffs.is_empty() { + println!("No test diffs found"); + return; + } + + // Sort diffs in decreasing order by diff count + diffs.sort_by_key(|entry| Reverse(entry.test_diffs.len())); + + fn format_outcome(outcome: &TestOutcome) -> String { + match outcome { + TestOutcome::Passed => "pass".to_string(), + TestOutcome::Failed => "fail".to_string(), + TestOutcome::Ignored { ignore_reason } => { + let reason = match ignore_reason { + Some(reason) => format!(" ({reason})"), + None => String::new(), + }; + format!("ignore{reason}") + } + } + } + + fn format_diff(diff: &TestOutcomeDiff) -> String { + match diff { + TestOutcomeDiff::ChangeOutcome { before, after } => { + format!("{} -> {}", format_outcome(before), format_outcome(after)) + } + TestOutcomeDiff::Missing { before } => { + format!("{} -> [missing]", format_outcome(before)) + } + TestOutcomeDiff::Added(outcome) => { + format!("[missing] -> {}", format_outcome(outcome)) + } + } + } + + let max_diff_count = 10; + let max_job_count = 5; + let max_test_count = 10; + + for diff in diffs.iter().take(max_diff_count) { + let mut jobs = diff.jobs.clone(); + jobs.sort(); + + let jobs = jobs.iter().take(max_job_count).map(|j| format!("`{j}`")).collect::>(); + + let extra_jobs = diff.jobs.len().saturating_sub(max_job_count); + let suffix = if extra_jobs > 0 { + format!(" (and {extra_jobs} {})", pluralize("other", extra_jobs)) + } else { + String::new() + }; + println!("- {}{suffix}", jobs.join(",")); + + let extra_tests = diff.test_diffs.len().saturating_sub(max_test_count); + for (test, outcome_diff) in diff.test_diffs.iter().take(max_test_count) { + println!(" - {}: {}", test.name, format_diff(&outcome_diff)); + } + if extra_tests > 0 { + println!(" - (and {extra_tests} additional {})", pluralize("tests", extra_tests)); + } + } + + let extra_diffs = diffs.len().saturating_sub(max_diff_count); + if extra_diffs > 0 { + println!("\n(and {extra_diffs} additional {})", pluralize("diff", extra_diffs)); + } +} + +fn pluralize(text: &str, count: usize) -> String { + if count == 1 { text.to_string() } else { format!("{text}s") } +} diff --git a/src/ci/citool/src/metrics.rs b/src/ci/citool/src/metrics.rs index 2386413ed6b1d..8548602b31cb7 100644 --- a/src/ci/citool/src/metrics.rs +++ b/src/ci/citool/src/metrics.rs @@ -105,17 +105,21 @@ struct TestSuiteRecord { failed: u64, } +fn test_metadata_name(metadata: &TestSuiteMetadata) -> String { + match metadata { + TestSuiteMetadata::CargoPackage { crates, stage, .. } => { + format!("{} (stage {stage})", crates.join(", ")) + } + TestSuiteMetadata::Compiletest { suite, stage, .. } => { + format!("{suite} (stage {stage})") + } + } +} + fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap { let mut records: BTreeMap = BTreeMap::new(); for suite in suites { - let name = match &suite.metadata { - TestSuiteMetadata::CargoPackage { crates, stage, .. } => { - format!("{} (stage {stage})", crates.join(", ")) - } - TestSuiteMetadata::Compiletest { suite, stage, .. } => { - format!("{suite} (stage {stage})") - } - }; + let name = test_metadata_name(&suite.metadata); let record = records.entry(name).or_default(); for test in &suite.tests { match test.outcome { @@ -134,7 +138,7 @@ fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap Vec<&TestSuite> { +pub fn get_test_suites(metrics: &JsonRoot) -> Vec<&TestSuite> { fn visit_test_suites<'a>(nodes: &'a [JsonNode], suites: &mut Vec<&'a TestSuite>) { for node in nodes { match node {