Merge branch 'master' into master

Author: eyraudh

.editorconfig

@@ -9,6 +9,8 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[!src/llvm-project]
 indent_style = space
 indent_size = 4
 

.git-blame-ignore-revs

@@ -29,3 +29,5 @@ ec2cc761bc7067712ecc7734502f703fe3b024c8
 99cb0c6bc399fb94a0ddde7e9b38e9c00d523bad
 # reformat with rustfmt edition 2024
 c682aa162b0d41e21cc6748f4fecfe01efb69d1f
+# reformat with updated edition 2024
+1fcae03369abb4c2cc180cd5a49e1f4440a81300

.github/ISSUE_TEMPLATE/bootstrap.md

@@ -0,0 +1,70 @@
+---
+name: Bootstrap (Rust Build System) Report
+about: Issues encountered on bootstrap build system
+labels: C-bug, T-bootstrap
+---
+
+<!--
+Thank you for submitting a bootstrap report! Please provide detailed information to help us reproduce and diagnose the issue.
+-->
+
+### Summary
+
+<!--
+Provide a brief description of the problem you are experiencing.
+-->
+
+### Command used
+
+```sh
+<command>
+```
+
+### Expected behaviour
+
+<!--
+Describe what you expected to happen.
+-->
+
+### Actual behaviour
+
+<!--
+Describe what actually happened.
+-->
+
+### Bootstrap configuration (bootstrap.toml)
+```toml
+<config>
+```
+
+### Operating system
+
+<!--
+e.g., Ubuntu 22.04, macOS 12, Windows 10
+-->
+
+### HEAD
+
+<!--
+Output of `git rev-parse HEAD` command, or content of the `git-commit-hash` file if using a tarball source.
+-->
+
+### Additional context
+<!--
+Include any other relevant information (e.g., if you have custom patches or modifications on the project).
+-->
+
+
+<!--
+Include the complete build log in the section below.
+Enable backtrace and verbose mode if possible for more detailed information e.g., with `RUST_BACKTRACE=1 ./x build -v`.
+-->
+<details><summary>Build Log</summary>
+<p>
+
+```txt
+<log>
+```
+
+</p>
+</details>

.github/workflows/ci.yml

@@ -1,8 +1,8 @@
 # This file defines our primary CI workflow that runs on pull requests
 # and also on pushes to special branches (auto, try).
 #
-# The actual definition of the executed jobs is calculated by a Python
-# script located at src/ci/github-actions/calculate-job-matrix.py, which
+# The actual definition of the executed jobs is calculated by the
+# `src/ci/citool` crate, which
 # uses job definition data from src/ci/github-actions/jobs.yml.
 # You should primarily modify the `jobs.yml` file if you want to modify
 # what jobs are executed in CI.
@@ -56,46 +56,32 @@ jobs:
       - name: Calculate the CI job matrix
         env:
           COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
-        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        run: |
+          cd src/ci/citool
+          CARGO_INCREMENTAL=0 cargo test
+          CARGO_INCREMENTAL=0 cargo run calculate-job-matrix >> $GITHUB_OUTPUT
         id: jobs
   job:
-    name: ${{ matrix.name }}
+    name: ${{ matrix.full_name }}
     needs: [ calculate_matrix ]
     runs-on: "${{ matrix.os }}"
-    defaults:
-      run:
-        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 360
     env:
-      CI_JOB_NAME: ${{ matrix.image }}
+      CI_JOB_NAME: ${{ matrix.name }}
+      CI_JOB_DOC_URL: ${{ matrix.doc_url }}
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
       HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
       DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       SCCACHE_BUCKET: rust-lang-ci-sccache2
+      SCCACHE_REGION: us-west-1
       CACHE_DOMAIN: ci-caches.rust-lang.org
     continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
         # Check the `calculate_matrix` job to see how is the matrix defined.
         include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
     steps:
-      - if: contains(matrix.os, 'windows')
-        uses: msys2/[email protected]
-        with:
-          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
-          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
-          # don't try to download updates for already installed packages
-          update: false
-          # don't try to use the msys that comes built-in to the github runner,
-          # so we can control what is installed (i.e. not python)
-          release: true
-          # Inherit the full path from the Windows environment, with MSYS2's */bin/
-          # dirs placed in front. This lets us run Windows-native Python etc.
-          path-type: inherit
-          install: >
-            make
-
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
 
@@ -109,7 +95,7 @@ jobs:
       # intensive jobs to run on free runners, which however also have
       # less disk space.
       - name: free up disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        run: src/ci/scripts/free-disk-space.sh
         if: matrix.free_disk
 
       # Rust Log Analyzer can't currently detect the PR number of a GitHub
@@ -192,9 +178,33 @@ jobs:
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
 
+      # Show the environment just before we run the build
+      # This makes it easier to diagnose problems with the above install scripts.
+      - name: show the current environment
+        run: src/ci/scripts/dump-environment.sh
+
+      # Pre-build citool before the following step uninstalls rustup
+      # Build it into the build directory, to avoid modifying sources
+      - name: build citool
+        run: |
+          cd src/ci/citool
+          CARGO_INCREMENTAL=0 CARGO_TARGET_DIR=../../../build/citool cargo build
+
       - name: run the build
-        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
-        run: src/ci/scripts/run-build-from-ci.sh 2>&1
+        run: |
+          set +e
+          # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
+          src/ci/scripts/run-build-from-ci.sh 2>&1
+          STATUS=$?
+          set -e
+
+          if [[ "$STATUS" -ne 0 && -n "$CI_JOB_DOC_URL" ]]; then
+            echo "****************************************************************************"
+            echo "To find more information about this job, visit the following URL:"
+            echo "$CI_JOB_DOC_URL"
+            echo "****************************************************************************"
+          fi
+          exit ${STATUS}
         env:
           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
@@ -228,16 +238,37 @@ jobs:
         # erroring about invalid credentials instead.
         if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
 
+      - name: postprocess metrics into the summary
+        # This step is not critical, and if some I/O problem happens, we don't want
+        # to cancel the build.
+        continue-on-error: true
+        run: |
+          if [ -f build/metrics.json ]; then
+            METRICS=build/metrics.json
+          elif [ -f obj/build/metrics.json ]; then
+            METRICS=obj/build/metrics.json
+          else
+            echo "No metrics.json found"
+            exit 0
+          fi
+
+          # Get closest bors merge commit
+          PARENT_COMMIT=`git rev-list --author='bors <[email protected]>' -n1 --first-parent HEAD^1`
+
+          ./build/citool/debug/citool postprocess-metrics \
+              --job-name ${CI_JOB_NAME} \
+              --parent ${PARENT_COMMIT} \
+              ${METRICS} >> ${GITHUB_STEP_SUMMARY}
+
       - name: upload job metrics to DataDog
+        # This step is not critical, and if some I/O problem happens, we don't want
+        # to cancel the build.
+        continue-on-error: true
         if: needs.calculate_matrix.outputs.run_type != 'pr'
         env:
-          DATADOG_SITE: datadoghq.com
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
-          DD_GITHUB_JOB_NAME: ${{ matrix.name }}
-        run: |
-          cd src/ci
-          npm ci
-          python3 scripts/upload-build-metrics.py ../../build/cpu-usage.csv
+          DD_GITHUB_JOB_NAME: ${{ matrix.full_name }}
+        run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv
 
   # This job isused to tell bors the final status of the build, as there is no practical way to detect
   # when a workflow is successful listening to webhooks only in our current bors implementation (homu).

.github/workflows/ghcr.yml

@@ -0,0 +1,75 @@
+# Mirror DockerHub images used by the Rust project to ghcr.io.
+# Images are available at https://github.com/orgs/rust-lang/packages.
+#
+# In some CI jobs, we pull images from ghcr.io instead of Docker Hub because
+# Docker Hub has a rate limit, while ghcr.io doesn't.
+# Those images are pushed to ghcr.io by this job.
+#
+# While Docker Hub rate limit *shouldn't* be an issue on GitHub Actions,
+# it certainly is for AWS codebuild.
+#
+# Note that authenticating to DockerHub or other registries isn't possible
+# for PR jobs, because forks can't access secrets.
+# That's why we use ghcr.io: it has no rate limit and it doesn't require authentication.
+
+name: GHCR image mirroring
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run daily at midnight UTC
+    - cron: '0 0 * * *'
+
+jobs:
+  mirror:
+    name: DockerHub mirror
+    runs-on: ubuntu-24.04
+    if: github.repository == 'rust-lang/rust'
+    permissions:
+      # Needed to write to the ghcr.io registry
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+
+      # Download crane in the current directory.
+      # We use crane because it copies the docker image for all the architectures available in
+      # DockerHub for the image.
+      # Learn more about crane at
+      # https://github.com/google/go-containerregistry/blob/main/cmd/crane/README.md
+      - name: Download crane
+        run: |
+          curl -sL "https://github.com/google/go-containerregistry/releases/download/${VERSION}/go-containerregistry_${OS}_${ARCH}.tar.gz" | tar -xzf -
+        env:
+          VERSION: v0.20.2
+          OS: Linux
+          ARCH: x86_64
+
+      - name: Mirror DockerHub
+        run: |
+          # List of DockerHub images to mirror to ghcr.io
+          images=(
+            # Mirrored because used by the mingw-check-tidy, which doesn't cache Docker images
+            "ubuntu:22.04"
+            # Mirrored because used by all linux CI jobs, including mingw-check-tidy
+            "moby/buildkit:buildx-stable-1"
+            # Mirrored because used when CI is running inside a Docker container
+            "alpine:3.4"
+            # Mirrored because used by dist-x86_64-linux
+            "centos:7"
+          )
+
+          # Mirror each image from DockerHub to ghcr.io
+          for img in "${images[@]}"; do
+            echo "Mirroring ${img}..."
+            # Remove namespace from the image if any.
+            # E.g. "moby/buildkit:buildx-stable-1" becomes "buildkit:buildx-stable-1"
+            dest_image=$(echo "${img}" | cut -d'/' -f2-)
+            ./crane copy \
+              "docker.io/${img}" \
+              "ghcr.io/${{ github.repository_owner }}/${dest_image}"
+          done

.github/workflows/post-merge.yml

@@ -0,0 +1,46 @@
+# Workflow that runs after a merge to master, analyses changes in test executions
+# and posts the result to the merged PR.
+
+name: Post merge analysis
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  analysis:
+    runs-on: ubuntu-24.04
+    if: github.repository == 'rust-lang/rust'
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Make sure that we have enough commits to find the parent merge commit.
+          # Since all merges should be through merge commits, fetching two commits
+          # should be enough to get the parent bors merge commit.
+          fetch-depth: 2
+      - name: Perform analysis and send PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Get closest bors merge commit
+          PARENT_COMMIT=`git rev-list --author='bors <[email protected]>' -n1 --first-parent HEAD^1`
+          echo "Parent: ${PARENT_COMMIT}"
+
+          # Find PR for the current commit
+          HEAD_PR=`gh pr list --search "${{ github.sha }}" --state merged --json number --jq '.[0].number'`
+          echo "HEAD: ${{ github.sha }} (#${HEAD_PR})"
+
+          cd src/ci/citool
+
+          printf "<details>\n<summary>What is this?</summary>\n" >> output.log
+          printf "This is an experimental post-merge analysis report that shows differences in test outcomes between the merged PR and its parent PR.\n" >> output.log
+          printf "</details>\n\n" >> output.log
+
+          cargo run --release post-merge-report ${PARENT_COMMIT} ${{ github.sha }} >> output.log
+
+          cat output.log
+
+          gh pr comment ${HEAD_PR} -F output.log

.gitignore

@@ -1,6 +1,6 @@
 # This file should only ignore things that are generated during a `x.py` build,
 # generated by common IDEs, and optional files controlled by the user that
-# affect the build (such as config.toml).
+# affect the build (such as bootstrap.toml).
 # In particular, things like `mir_dump` should not be listed here; they are only
 # created during manual debugging and many people like to clean up instead of
 # having git ignore such leftovers. You can use `.git/info/exclude` to
@@ -34,6 +34,7 @@ Session.vim
 !/tests/run-make/thumb-none-qemu/example/.cargo
 
 ## Configuration
+/bootstrap.toml
 /config.toml
 /Makefile
 config.mk
@@ -53,7 +54,10 @@ no_llvm_build
 /target
 /library/target
 /src/bootstrap/target
+/src/ci/citool/target
 /src/tools/x/target
+# Created by `x vendor`
+/vendor
 # Created by default with `src/ci/docker/run.sh`
 /obj/
 # Created by nix dev shell / .envrc
@@ -83,6 +87,7 @@ __pycache__/
 node_modules
 package-lock.json
 package.json
+/src/doc/rustc-dev-guide/mermaid.min.js
 
 ## Rustdoc GUI tests
 tests/rustdoc-gui/src/**.lock