From f30e9c943ccd1eacb6a75d5b0a64d7091e1a8e83 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Tue, 10 Jun 2025 10:30:06 +0200 Subject: [PATCH] ci: split x86_64-gnu-tools job --- .../host-x86_64/x86_64-gnu-miri/Dockerfile | 57 +++++++++++++++++ .../host-x86_64/x86_64-gnu-miri/check-miri.sh | 62 +++++++++++++++++++ .../x86_64-gnu-tools/checktools.sh | 55 ---------------- src/ci/github-actions/jobs.yml | 7 ++- 4 files changed, 125 insertions(+), 56 deletions(-) create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile create mode 100755 src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile new file mode 100644 index 0000000000000..b937bc3e678d3 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile @@ -0,0 +1,57 @@ +FROM ghcr.io/rust-lang/ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + libssl-dev \ + sudo \ + xz-utils \ + tidy \ + \ + libc6 \ + wget \ + # libgccjit dependencies + flex \ + libmpfr-dev \ + libgmp-dev \ + libmpc3 \ + libmpc-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" + +COPY host-x86_64/x86_64-gnu-miri/check-miri.sh /tmp/ + +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --enable-new-symbol-mangling + +ENV HOST_TARGET x86_64-unknown-linux-gnu + +# FIXME(#133381): currently rustc alt builds do *not* have rustc debug +# assertions enabled! Therefore, we cannot force download CI rustc. +#ENV FORCE_CI_RUSTC 1 + +COPY scripts/shared.sh /scripts/ + +# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries +# to create a new folder. For reference: +# https://github.com/puppeteer/puppeteer/issues/375 +# +# We also specify the version in case we need to update it to go around cache limitations. +# +# The `browser-ui-test.version` file is also used by bootstrap to emit warnings in case +# the local version of the package is different than the one used by the CI. +ENV SCRIPT /tmp/check-miri.sh ../x.py diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh b/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh new file mode 100755 index 0000000000000..c2a5b308b3284 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# ignore-tidy-linelength + +set -eu +set -x # so one can see where we are in the script + +X_PY="$1" + +# Testing Miri is a bit complicated. +# We set the GC interval to the shortest possible value (0 would be off) to increase the chance +# that bugs which only surface when the GC runs at a specific time are more likely to cause CI to fail. +# This significantly increases the runtime of our test suite, or we'd do this in PR CI too. +if [ -z "${PR_CI_JOB:-}" ]; then + MIRIFLAGS=-Zmiri-provenance-gc=1 python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri +else + python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri +fi +# We re-run the test suite for a chance to find bugs in the intrinsic fallback bodies and in MIR +# optimizations. This can miss UB, so we only run the "pass" tests. We need to enable debug +# assertions as `-O` disables them but some tests rely on them. We also set a cfg flag so tests can +# adjust their expectations if needed. This can change the output of the tests so we ignore that, +# we only ensure that all assertions still pass. +MIRIFLAGS="-Zmiri-force-intrinsic-fallback --cfg force_intrinsic_fallback -O -Zmir-opt-level=4 -Cdebug-assertions=yes" \ + MIRI_SKIP_UI_CHECKS=1 \ + python3 "$X_PY" test --stage 2 src/tools/miri -- tests/pass tests/panic +# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. +# Also cover some other targets via cross-testing, in particular all tier 1 targets. +case $HOST_TARGET in + x86_64-unknown-linux-gnu) + # Only this branch runs in PR CI. + # Fully test all main OSes, and all main architectures. + python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target aarch64-apple-darwin + python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target i686-pc-windows-msvc + # Only run "pass" tests for the remaining targets, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass + ;; + x86_64-pc-windows-msvc) + # Strangely, Linux targets do not work here. cargo always says + # "error: cannot produce cdylib for ... as the target ... does not support these crate types". + # Only run "pass" tests, which is quite a bit faster. + #FIXME: Re-enable this once CI issues are fixed + # See + # For now, these tests are moved to `x86_64-msvc-ext2` in `src/ci/github-actions/jobs.yml`. + #python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin --test-args pass + ;; + *) + echo "FATAL: unexpected host $HOST_TARGET" + exit 1 + ;; +esac +# Also smoke-test `x.py miri`. This doesn't run any actual tests (that would take too long), +# but it ensures that the crates build properly when tested with Miri. + +#FIXME: Re-enable this for msvc once CI issues are fixed +if [ "$HOST_TARGET" != "x86_64-pc-windows-msvc" ]; then + python3 "$X_PY" miri --stage 2 library/core --test-args notest + python3 "$X_PY" miri --stage 2 library/alloc --test-args notest + python3 "$X_PY" miri --stage 2 library/std --test-args notest +fi diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 62e0451814b35..ff9fedad6567d 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -30,58 +30,3 @@ cat /tmp/toolstate/toolstates.json python3 "$X_PY" test --stage 2 check-tools python3 "$X_PY" test --stage 2 src/tools/clippy python3 "$X_PY" test --stage 2 src/tools/rustfmt - -# Testing Miri is a bit more complicated. -# We set the GC interval to the shortest possible value (0 would be off) to increase the chance -# that bugs which only surface when the GC runs at a specific time are more likely to cause CI to fail. -# This significantly increases the runtime of our test suite, or we'd do this in PR CI too. -if [ -z "${PR_CI_JOB:-}" ]; then - MIRIFLAGS=-Zmiri-provenance-gc=1 python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri -else - python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri -fi -# We re-run the test suite for a chance to find bugs in the intrinsic fallback bodies and in MIR -# optimizations. This can miss UB, so we only run the "pass" tests. We need to enable debug -# assertions as `-O` disables them but some tests rely on them. We also set a cfg flag so tests can -# adjust their expectations if needed. This can change the output of the tests so we ignore that, -# we only ensure that all assertions still pass. -MIRIFLAGS="-Zmiri-force-intrinsic-fallback --cfg force_intrinsic_fallback -O -Zmir-opt-level=4 -Cdebug-assertions=yes" \ - MIRI_SKIP_UI_CHECKS=1 \ - python3 "$X_PY" test --stage 2 src/tools/miri -- tests/pass tests/panic -# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. -# Also cover some other targets via cross-testing, in particular all tier 1 targets. -case $HOST_TARGET in - x86_64-unknown-linux-gnu) - # Only this branch runs in PR CI. - # Fully test all main OSes, and all main architectures. - python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target aarch64-apple-darwin - python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target i686-pc-windows-msvc - # Only run "pass" tests for the remaining targets, which is quite a bit faster. - python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass - python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass - python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass - python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass - ;; - x86_64-pc-windows-msvc) - # Strangely, Linux targets do not work here. cargo always says - # "error: cannot produce cdylib for ... as the target ... does not support these crate types". - # Only run "pass" tests, which is quite a bit faster. - #FIXME: Re-enable this once CI issues are fixed - # See - # For now, these tests are moved to `x86_64-msvc-ext2` in `src/ci/github-actions/jobs.yml`. - #python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin --test-args pass - ;; - *) - echo "FATAL: unexpected host $HOST_TARGET" - exit 1 - ;; -esac -# Also smoke-test `x.py miri`. This doesn't run any actual tests (that would take too long), -# but it ensures that the crates build properly when tested with Miri. - -#FIXME: Re-enable this for msvc once CI issues are fixed -if [ "$HOST_TARGET" != "x86_64-pc-windows-msvc" ]; then - python3 "$X_PY" miri --stage 2 library/core --test-args notest - python3 "$X_PY" miri --stage 2 library/alloc --test-args notest - python3 "$X_PY" miri --stage 2 library/std --test-args notest -fi diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 43c77d1ddf7f5..cb7d9c0f86f84 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -150,7 +150,9 @@ pr: DOCKER_SCRIPT: stage_2_test_set2.sh <<: *job-aarch64-linux - name: x86_64-gnu-tools - <<: *job-linux-36c-codebuild + <<: *job-linux-4c + - name: x86_64-gnu-miri + <<: *job-linux-4c # Jobs that run when you perform a try build (@bors try) # These jobs automatically inherit envs.try, to avoid repeating @@ -419,6 +421,9 @@ auto: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-4c + - name: x86_64-gnu-miri + <<: *job-linux-4c + #################### # macOS Builders # ####################