From 90745bd1ededab96175485b098f58ae6d26e24f8 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 15 Sep 2024 07:41:24 +0300 Subject: [PATCH 1/4] Feed `GitInfo`s from `Config` to `Build` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 22 ++++++++++++++++++++++ src/bootstrap/src/lib.rs | 19 +++++++++---------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 79c2f73161e48..928366e64d0ad 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -342,6 +342,15 @@ pub struct Config { pub out: PathBuf, pub rust_info: channel::GitInfo, + pub cargo_info: channel::GitInfo, + pub rust_analyzer_info: channel::GitInfo, + pub clippy_info: channel::GitInfo, + pub miri_info: channel::GitInfo, + pub rustfmt_info: channel::GitInfo, + pub enzyme_info: channel::GitInfo, + pub in_tree_llvm_info: channel::GitInfo, + pub in_tree_gcc_info: channel::GitInfo, + // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, @@ -1797,6 +1806,19 @@ impl Config { config.omit_git_hash = omit_git_hash.unwrap_or(default); config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); + config.cargo_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/cargo")); + config.rust_analyzer_info = + GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer")); + config.clippy_info = + GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/clippy")); + config.miri_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/miri")); + config.rustfmt_info = + GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rustfmt")); + config.enzyme_info = + GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/enzyme")); + config.in_tree_llvm_info = GitInfo::new(false, &config.src.join("src/llvm-project")); + config.in_tree_gcc_info = GitInfo::new(false, &config.src.join("src/gcc")); + // We need to override `rust.channel` if it's manually specified when using the CI rustc. // This is because if the compiler uses a different channel than the one specified in config.toml, // tests may fail due to using a different channel than the one used by the compiler during tests. diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index df2acd8b76bd3..6467f4e35910d 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -305,18 +305,17 @@ impl Build { #[cfg(not(unix))] let is_sudo = false; - let omit_git_hash = config.omit_git_hash; - let rust_info = GitInfo::new(omit_git_hash, &src); - let cargo_info = GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); - let rust_analyzer_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); - let clippy_info = GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); - let miri_info = GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); - let rustfmt_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); - let enzyme_info = GitInfo::new(omit_git_hash, &src.join("src/tools/enzyme")); + let rust_info = config.rust_info.clone(); + let cargo_info = config.cargo_info.clone(); + let rust_analyzer_info = config.rust_analyzer_info.clone(); + let clippy_info = config.clippy_info.clone(); + let miri_info = config.miri_info.clone(); + let rustfmt_info = config.rustfmt_info.clone(); + let enzyme_info = config.enzyme_info.clone(); // we always try to use git for LLVM builds - let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project")); - let in_tree_gcc_info = GitInfo::new(false, &src.join("src/gcc")); + let in_tree_llvm_info = config.in_tree_llvm_info.clone(); + let in_tree_gcc_info = config.in_tree_gcc_info.clone(); let initial_target_libdir_str = if config.dry_run() { "/dummy/lib/path/to/lib/".to_string() From 6658c8e2449fb6a674ac606823e43c44f3caf275 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 15 Sep 2024 07:43:16 +0300 Subject: [PATCH 2/4] check if the LLVM submodule is fetched in `is_ci_llvm_modified` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 12 +++++++++++- src/bootstrap/src/lib.rs | 2 -- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 08e24ecc3401a..81a48000cf145 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -241,7 +241,17 @@ pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { /// Returns true if we're running in CI with modified LLVM (and thus can't download it) pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { + // If the LLVM submodule is unavailable (which is the case when `llvm.download-ci-llvm` is set to true), + // LLVM cannot be modified which means it is unnecessary to run the git logic below. + // + // This is very unlikely to happen on our (rust-lang/rust) CI runners, as we intentionally fetch all + // submodules in CI and most of the time (probably always) prefer `llvm.download-ci-llvm` to be set + // to "if-unchanged" or true. + if config.in_tree_llvm_info.is_managed_git_subrepository() { + return false; + } + + CiEnv::is_ci() && { // We assume we have access to git, so it's okay to unconditionally pass // `true` here. let llvm_sha = detect_llvm_sha(config, true); diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 6467f4e35910d..9a48b7d36d9a4 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -312,8 +312,6 @@ impl Build { let miri_info = config.miri_info.clone(); let rustfmt_info = config.rustfmt_info.clone(); let enzyme_info = config.enzyme_info.clone(); - - // we always try to use git for LLVM builds let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); From 2d85139f87f52f28172be61c2957d8f856b3d9ac Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 25 Sep 2024 19:48:15 +0300 Subject: [PATCH 3/4] create `CiEnv::is_rust_lang_managed_ci_job` Signed-off-by: onur-ozkan --- src/tools/build_helper/src/ci.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/tools/build_helper/src/ci.rs b/src/tools/build_helper/src/ci.rs index 6d79c7c83ad8b..60f319129a0bd 100644 --- a/src/tools/build_helper/src/ci.rs +++ b/src/tools/build_helper/src/ci.rs @@ -19,6 +19,15 @@ impl CiEnv { pub fn is_ci() -> bool { Self::current() != CiEnv::None } + + /// Checks if running in rust-lang/rust managed CI job. + pub fn is_rust_lang_managed_ci_job() -> bool { + Self::is_ci() + // If both are present, we can assume it's an upstream CI job + // as they are always set unconditionally. + && std::env::var_os("CI_JOB_NAME").is_some() + && std::env::var_os("TOOLSTATE_REPO").is_some() + } } pub mod gha { From 5840d8791c1fc464e0aeca0690bff95274ddacd7 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 25 Sep 2024 19:50:42 +0300 Subject: [PATCH 4/4] in `llvm::is_ci_llvm_modified`, check if it's rust-lang/rust CI job Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 81a48000cf145..400297316bb43 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -241,17 +241,7 @@ pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { /// Returns true if we're running in CI with modified LLVM (and thus can't download it) pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - // If the LLVM submodule is unavailable (which is the case when `llvm.download-ci-llvm` is set to true), - // LLVM cannot be modified which means it is unnecessary to run the git logic below. - // - // This is very unlikely to happen on our (rust-lang/rust) CI runners, as we intentionally fetch all - // submodules in CI and most of the time (probably always) prefer `llvm.download-ci-llvm` to be set - // to "if-unchanged" or true. - if config.in_tree_llvm_info.is_managed_git_subrepository() { - return false; - } - - CiEnv::is_ci() && { + CiEnv::is_rust_lang_managed_ci_job() && config.rust_info.is_managed_git_subrepository() && { // We assume we have access to git, so it's okay to unconditionally pass // `true` here. let llvm_sha = detect_llvm_sha(config, true);