From 24b495d378ea97214abef1f1c8c6f54fb788dbd7 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Fri, 18 Jul 2025 14:50:34 +0530 Subject: [PATCH 1/2] This PR includes the following changes: 1. Moves TOML build parsing into a separate module. 2. Corrects the ordering of `apply_*_config` method calls. 3. Moves download logic specific to `toml_config` into its own module. 4. Updates tests to align with the correct behavior. --- src/bootstrap/src/core/config/config.rs | 615 +++----------------- src/bootstrap/src/core/config/tests.rs | 3 +- src/bootstrap/src/core/config/toml/build.rs | 394 ++++++++++++- src/bootstrap/src/core/config/toml/llvm.rs | 93 ++- src/bootstrap/src/core/config/toml/rust.rs | 181 ++++-- src/bootstrap/src/core/download.rs | 174 ++---- 6 files changed, 748 insertions(+), 712 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 6e04f11542438..3b0652ff6c542 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -17,8 +17,7 @@ use std::cell::Cell; use std::collections::{BTreeSet, HashMap, HashSet}; use std::io::IsTerminal; -use std::path::{Path, PathBuf, absolute}; -use std::str::FromStr; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::{cmp, env, fs}; @@ -29,13 +28,10 @@ use serde::Deserialize; #[cfg(feature = "tracing")] use tracing::{instrument, span}; -use crate::core::build_steps::llvm; -use crate::core::build_steps::llvm::LLVM_INVALIDATION_PATHS; pub use crate::core::config::flags::Subcommand; use crate::core::config::flags::{Color, Flags}; -use crate::core::config::target_selection::TargetSelectionList; use crate::core::config::toml::TomlConfig; -use crate::core::config::toml::build::{Build, Tool}; +use crate::core::config::toml::build::Tool; use crate::core::config::toml::change_id::ChangeId; use crate::core::config::toml::rust::{ LldMode, RustOptimize, check_incompatible_options_for_ci_rustc, @@ -43,35 +39,13 @@ use crate::core::config::toml::rust::{ use crate::core::config::toml::target::Target; use crate::core::config::{ DebuginfoLevel, DryRun, GccCiMode, LlvmLibunwind, Merge, ReplaceOpt, RustcLto, SplitDebuginfo, - StringOrBool, set, threads_from_config, + StringOrBool, threads_from_config, }; -use crate::core::download::is_download_ci_available; use crate::utils::channel; -use crate::utils::exec::{ExecutionContext, command}; +use crate::utils::exec::ExecutionContext; use crate::utils::helpers::{exe, get_host_target}; use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t}; -/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. -/// This means they can be modified and changes to these paths should never trigger a compiler build -/// when "if-unchanged" is set. -/// -/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during -/// the diff check. -/// -/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build -/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. -/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the -/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. -#[rustfmt::skip] // We don't want rustfmt to oneline this list -pub const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ - ":!library", - ":!src/tools", - ":!src/librustdoc", - ":!src/rustdoc-json-types", - ":!tests", - ":!triagebot.toml", -]; - /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `bootstrap.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -446,6 +420,13 @@ impl Config { config.exec_ctx = exec_ctx; + let read_toml = |path: &Path| { + get_toml(path).unwrap_or_else(|e| { + eprintln!("ERROR: Failed to parse '{}': {e}", path.display()); + exit!(2); + }) + }; + // Set flags. config.paths = std::mem::take(&mut flags_paths); @@ -487,6 +468,10 @@ impl Config { config.is_running_on_ci = flags_ci.unwrap_or(CiEnv::is_ci()); config.skip_std_check_if_no_download_rustc = flags_skip_std_check_if_no_download_rustc; + if let Some(flags_jobs) = flags_jobs { + config.jobs = Some(threads_from_config(flags_jobs)); + } + // Infer the rest of the configuration. if let Some(src) = flags_src { @@ -580,10 +565,7 @@ impl Config { } else { toml_path.clone() }); - get_toml(&toml_path).unwrap_or_else(|e| { - eprintln!("ERROR: Failed to parse '{}': {e}", toml_path.display()); - exit!(2); - }) + read_toml(&toml_path) } else { config.config = None; TomlConfig::default() @@ -612,10 +594,7 @@ impl Config { for include_path in toml.include.clone().unwrap_or_default().iter().rev() { let include_path = toml_path.parent().unwrap().join(include_path); - let included_toml = get_toml(&include_path).unwrap_or_else(|e| { - eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); - exit!(2); - }); + let included_toml = read_toml(&include_path); toml.merge( Some(include_path), &mut Default::default(), @@ -638,13 +617,7 @@ impl Config { include_path.push("bootstrap"); include_path.push("defaults"); include_path.push(format!("bootstrap.{include}.toml")); - let included_toml = get_toml(&include_path).unwrap_or_else(|e| { - eprintln!( - "ERROR: Failed to parse default config profile at '{}': {e}", - include_path.display() - ); - exit!(2); - }); + let included_toml = read_toml(&include_path); toml.merge( Some(include_path), &mut Default::default(), @@ -696,204 +669,20 @@ impl Config { config.change_id = toml.change_id.inner; - let Build { - description, - build, - host, - target, - build_dir, - cargo, - rustc, - rustfmt, - cargo_clippy, - docs, - compiler_docs, - library_docs_private_items, - docs_minification, - submodules, - gdb, - lldb, - nodejs, - npm, - python, - reuse, - locked_deps, - vendor, - full_bootstrap, - bootstrap_cache_path, - extended, - tools, - tool, - verbose, - sanitizers, - profiler, - cargo_native_static, - low_priority, - configure_args, - local_rebuild, - print_step_timings, - print_step_rusage, - check_stage, - doc_stage, - build_stage, - test_stage, - install_stage, - dist_stage, - bench_stage, - patch_binaries_for_nix, - // This field is only used by bootstrap.py - metrics: _, - android_ndk, - optimized_compiler_builtins, - jobs, - compiletest_diff_tool, - compiletest_use_stage0_libtest, - tidy_extra_checks, - ccache, - exclude, - } = toml.build.unwrap_or_default(); - - let mut paths: Vec = flags_skip.into_iter().chain(flags_exclude).collect(); - - if let Some(exclude) = exclude { - paths.extend(exclude); - } - - config.skip = paths - .into_iter() - .map(|p| { - // Never return top-level path here as it would break `--skip` - // logic on rustc's internal test framework which is utilized - // by compiletest. - if cfg!(windows) { - PathBuf::from(p.to_str().unwrap().replace('/', "\\")) - } else { - p - } - }) - .collect(); - - config.jobs = Some(threads_from_config(flags_jobs.unwrap_or(jobs.unwrap_or(0)))); - - if let Some(flags_build) = flags_build { - config.host_target = TargetSelection::from_user(&flags_build); - } else if let Some(file_build) = build { - config.host_target = TargetSelection::from_user(&file_build); - }; - - set(&mut config.out, flags_build_dir.or_else(|| build_dir.map(PathBuf::from))); - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = absolute(&config.out).expect("can't make empty path absolute"); - } - - if cargo_clippy.is_some() && rustc.is_none() { - println!( - "WARNING: Using `build.cargo-clippy` without `build.rustc` usually fails due to toolchain conflict." - ); - } - - config.initial_rustc = if let Some(rustc) = rustc { - if !flags_skip_stage0_validation { - config.check_stage0_version(&rustc, "rustc"); - } - rustc - } else { - config.download_beta_toolchain(); - config - .out - .join(config.host_target) - .join("stage0") - .join("bin") - .join(exe("rustc", config.host_target)) - }; - - config.initial_sysroot = t!(PathBuf::from_str( - command(&config.initial_rustc) - .args(["--print", "sysroot"]) - .run_in_dry_run() - .run_capture_stdout(&config) - .stdout() - .trim() - )); - - config.initial_cargo_clippy = cargo_clippy; - - config.initial_cargo = if let Some(cargo) = cargo { - if !flags_skip_stage0_validation { - config.check_stage0_version(&cargo, "cargo"); - } - cargo - } else { - config.download_beta_toolchain(); - config.initial_sysroot.join("bin").join(exe("cargo", config.host_target)) - }; - - // NOTE: it's important this comes *after* we set `initial_rustc` just above. - if config.dry_run() { - let dir = config.out.join("tmp-dry-run"); - t!(fs::create_dir_all(&dir)); - config.out = dir; - } - - config.hosts = if let Some(TargetSelectionList(arg_host)) = flags_host { - arg_host - } else if let Some(file_host) = host { - file_host.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - vec![config.host_target] - }; - config.targets = if let Some(TargetSelectionList(arg_target)) = flags_target { - arg_target - } else if let Some(file_target) = target { - file_target.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - // If target is *not* configured, then default to the host - // toolchains. - config.hosts.clone() - }; - - config.nodejs = nodejs.map(PathBuf::from); - config.npm = npm.map(PathBuf::from); - config.gdb = gdb.map(PathBuf::from); - config.lldb = lldb.map(PathBuf::from); - config.python = python.map(PathBuf::from); - config.reuse = reuse.map(PathBuf::from); - config.submodules = submodules; - config.android_ndk = android_ndk; - config.bootstrap_cache_path = bootstrap_cache_path; - set(&mut config.low_priority, low_priority); - set(&mut config.compiler_docs, compiler_docs); - set(&mut config.library_docs_private_items, library_docs_private_items); - set(&mut config.docs_minification, docs_minification); - set(&mut config.docs, docs); - set(&mut config.locked_deps, locked_deps); - set(&mut config.full_bootstrap, full_bootstrap); - set(&mut config.extended, extended); - config.tools = tools; - set(&mut config.tool, tool); - set(&mut config.verbose, verbose); - set(&mut config.sanitizers, sanitizers); - set(&mut config.profiler, profiler); - set(&mut config.cargo_native_static, cargo_native_static); - set(&mut config.configure_args, configure_args); - set(&mut config.local_rebuild, local_rebuild); - set(&mut config.print_step_timings, print_step_timings); - set(&mut config.print_step_rusage, print_step_rusage); - config.patch_binaries_for_nix = patch_binaries_for_nix; - config.verbose = cmp::max(config.verbose, flags_verbose as usize); // Verbose flag is a good default for `rust.verbose-tests`. config.verbose_tests = config.is_verbose(); - config.apply_install_config(toml.install); - config.llvm_assertions = toml.llvm.as_ref().is_some_and(|llvm| llvm.assertions.unwrap_or(false)); + if let Some(flags_build) = flags_build { + config.host_target = TargetSelection::from_user(&flags_build); + } else if let Some(Some(build)) = toml.build.as_ref().map(|build| build.build.clone()) { + config.host_target = TargetSelection::from_user(&build); + } + let file_content = t!(fs::read_to_string(config.src.join("src/ci/channel"))); let ci_channel = file_content.trim_end(); @@ -929,11 +718,38 @@ impl Config { config.in_tree_llvm_info = config.git_info(false, &config.src.join("src/llvm-project")); config.in_tree_gcc_info = config.git_info(false, &config.src.join("src/gcc")); - config.vendor = vendor.unwrap_or( - config.rust_info.is_from_tarball() - && config.src.join("vendor").exists() - && config.src.join(".cargo/config.toml").exists(), - ); + toml.rust.as_ref().map(|rust| { + // FIXME(#133381): alt rustc builds currently do *not* have rustc debug assertions + // enabled. We should not download a CI alt rustc if we need rustc to have debug + // assertions (e.g. for crashes test suite). This can be changed once something like + // [Enable debug assertions on alt + // builds](https://github.com/rust-lang/rust/pull/131077) lands. + // + // Note that `rust.debug = true` currently implies `rust.debug-assertions = true`! + // + // This relies also on the fact that the global default for `download-rustc` will be + // `false` if it's not explicitly set. + let debug_assertions_requested = matches!(rust.rustc_debug_assertions, Some(true)) + || (matches!(rust.debug, Some(true)) + && !matches!(rust.rustc_debug_assertions, Some(false))); + + if debug_assertions_requested + && let Some(ref opt) = rust.download_rustc + && opt.is_string_or_true() + { + eprintln!( + "WARN: currently no CI rustc builds have rustc debug assertions \ + enabled. Please either set `rust.debug-assertions` to `false` if you \ + want to use download CI rustc or set `rust.download-rustc` to `false`." + ); + } + + config.download_rustc_commit = config.download_ci_rustc_commit( + rust.download_rustc.clone(), + debug_assertions_requested, + config.llvm_assertions, + ); + }); if !is_user_configured_rust_channel && config.rust_info.is_from_tarball() { config.channel = ci_channel.into(); @@ -942,11 +758,7 @@ impl Config { config.rust_profile_use = flags_rust_profile_use; config.rust_profile_generate = flags_rust_profile_generate; - config.apply_target_config(toml.target); - config.apply_rust_config(toml.rust, flags_warnings); - config.reproducible_artifacts = flags_reproducible_artifact; - config.description = description; // We need to override `rust.channel` if it's manually specified when using the CI rustc. // This is because if the compiler uses a different channel than the one specified in bootstrap.toml, @@ -964,17 +776,40 @@ impl Config { config.channel = channel; } - config.apply_llvm_config(toml.llvm); + config.explicit_stage_from_cli = flags_stage.is_some(); + + config.skip.extend( + flags_skip + .into_iter() + .chain(flags_exclude) + .map(|p| { + // Never return top-level path here as it would break `--skip` + // logic on rustc's internal test framework which is utilized + // by compiletest. + if cfg!(windows) { + PathBuf::from(p.to_str().unwrap().replace('/', "\\")) + } else { + p + } + }) + .collect::>(), + ); + config.apply_install_config(toml.install); config.apply_gcc_config(toml.gcc); + config.apply_dist_config(toml.dist); - match ccache { - Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), - Some(StringOrBool::Bool(true)) => { - config.ccache = Some("ccache".to_string()); - } - Some(StringOrBool::Bool(false)) | None => {} - } + config.apply_build_config( + toml.build, + flags_skip_stage0_validation, + flags_stage, + flags_host, + flags_target, + flags_build_dir, + ); + config.apply_target_config(toml.target); + config.apply_rust_config(toml.rust, flags_warnings); + config.apply_llvm_config(toml.llvm); if config.llvm_from_ci { let triple = &config.host_target.triple; @@ -992,11 +827,6 @@ impl Config { Some(ci_llvm_bin.join(exe("FileCheck", config.host_target))); } - config.apply_dist_config(toml.dist); - - config.initial_rustfmt = - if let Some(r) = rustfmt { Some(r) } else { config.maybe_download_rustfmt() }; - if matches!(config.lld_mode, LldMode::SelfContained) && !config.lld_enabled && flags_stage.unwrap_or(0) > 0 @@ -1010,48 +840,6 @@ impl Config { panic!("Cannot enable LLD with `rust.lld = true` when using external llvm-config."); } - config.optimized_compiler_builtins = - optimized_compiler_builtins.unwrap_or(config.channel != "dev"); - config.compiletest_diff_tool = compiletest_diff_tool; - config.compiletest_use_stage0_libtest = compiletest_use_stage0_libtest.unwrap_or(true); - config.tidy_extra_checks = tidy_extra_checks; - - let download_rustc = config.download_rustc_commit.is_some(); - config.explicit_stage_from_cli = flags_stage.is_some(); - config.explicit_stage_from_config = test_stage.is_some() - || build_stage.is_some() - || doc_stage.is_some() - || dist_stage.is_some() - || install_stage.is_some() - || check_stage.is_some() - || bench_stage.is_some(); - - config.stage = match config.cmd { - Subcommand::Check { .. } => flags_stage.or(check_stage).unwrap_or(1), - Subcommand::Clippy { .. } | Subcommand::Fix => flags_stage.or(check_stage).unwrap_or(1), - // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. - Subcommand::Doc { .. } => { - flags_stage.or(doc_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Build => { - flags_stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Test { .. } | Subcommand::Miri { .. } => { - flags_stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Bench { .. } => flags_stage.or(bench_stage).unwrap_or(2), - Subcommand::Dist => flags_stage.or(dist_stage).unwrap_or(2), - Subcommand::Install => flags_stage.or(install_stage).unwrap_or(2), - Subcommand::Perf { .. } => flags_stage.unwrap_or(1), - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Subcommand::Clean { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Vendor { .. } => flags_stage.unwrap_or(0), - }; - // Now check that the selected stage makes sense, and if not, print a warning and end match (config.stage, &config.cmd) { (0, Subcommand::Build) => { @@ -1065,13 +853,6 @@ impl Config { _ => {} } - if config.compile_time_deps && !matches!(config.cmd, Subcommand::Check { .. }) { - eprintln!( - "WARNING: Can't use --compile-time-deps with any subcommand other than check." - ); - exit!(1); - } - // CI should always run stage 2 builds, unless it specifically states otherwise #[cfg(not(test))] if flags_stage.is_none() && config.is_running_on_ci { @@ -1148,47 +929,6 @@ impl Config { git.run_capture_stdout(self).stdout() } - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. - /// Return the version it would have used for the given commit. - pub(crate) fn artifact_version_part(&self, commit: &str) -> String { - let (channel, version) = if self.rust_info.is_managed_git_subrepository() { - let channel = - self.read_file_by_commit(Path::new("src/ci/channel"), commit).trim().to_owned(); - let version = - self.read_file_by_commit(Path::new("src/version"), commit).trim().to_owned(); - (channel, version) - } else { - let channel = fs::read_to_string(self.src.join("src/ci/channel")); - let version = fs::read_to_string(self.src.join("src/version")); - match (channel, version) { - (Ok(channel), Ok(version)) => { - (channel.trim().to_owned(), version.trim().to_owned()) - } - (channel, version) => { - let src = self.src.display(); - eprintln!("ERROR: failed to determine artifact channel and/or version"); - eprintln!( - "HELP: consider using a git checkout or ensure these files are readable" - ); - if let Err(channel) = channel { - eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); - } - if let Err(version) = version { - eprintln!("reading {src}/src/version failed: {version:?}"); - } - panic!(); - } - } - }; - - match channel.as_str() { - "stable" => version, - "beta" => channel, - "nightly" => channel, - other => unreachable!("{:?} is not recognized as a valid channel", other), - } - } - /// Try to find the relative path of `bindir`, otherwise return it in full. pub fn bindir_relative(&self) -> &Path { let bindir = &self.bindir; @@ -1491,187 +1231,6 @@ impl Config { } } - #[cfg(test)] - pub fn check_stage0_version(&self, _program_path: &Path, _component_name: &'static str) {} - - /// check rustc/cargo version is same or lower with 1 apart from the building one - #[cfg(not(test))] - pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { - use build_helper::util::fail; - - if self.dry_run() { - return; - } - - let stage0_output = - command(program_path).arg("--version").run_capture_stdout(self).stdout(); - let mut stage0_output = stage0_output.lines().next().unwrap().split(' '); - - let stage0_name = stage0_output.next().unwrap(); - if stage0_name != component_name { - fail(&format!( - "Expected to find {component_name} at {} but it claims to be {stage0_name}", - program_path.display() - )); - } - - let stage0_version = - semver::Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) - .unwrap(); - let source_version = semver::Version::parse( - fs::read_to_string(self.src.join("src/version")).unwrap().trim(), - ) - .unwrap(); - if !(source_version == stage0_version - || (source_version.major == stage0_version.major - && (source_version.minor == stage0_version.minor - || source_version.minor == stage0_version.minor + 1))) - { - let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); - fail(&format!( - "Unexpected {component_name} version: {stage0_version}, we should use {prev_version}/{source_version} to build source with {source_version}" - )); - } - } - - /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - pub fn download_ci_rustc_commit( - &self, - download_rustc: Option, - debug_assertions_requested: bool, - llvm_assertions: bool, - ) -> Option { - if !is_download_ci_available(&self.host_target.triple, llvm_assertions) { - return None; - } - - // If `download-rustc` is not set, default to rebuilding. - let if_unchanged = match download_rustc { - // Globally default `download-rustc` to `false`, because some contributors don't use - // profiles for reasons such as: - // - They need to seamlessly switch between compiler/library work. - // - They don't want to use compiler profile because they need to override too many - // things and it's easier to not use a profile. - None | Some(StringOrBool::Bool(false)) => return None, - Some(StringOrBool::Bool(true)) => false, - Some(StringOrBool::String(s)) if s == "if-unchanged" => { - if !self.rust_info.is_managed_git_subrepository() { - println!( - "ERROR: `download-rustc=if-unchanged` is only compatible with Git managed sources." - ); - crate::exit!(1); - } - - true - } - Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {other}") - } - }; - - let commit = if self.rust_info.is_managed_git_subrepository() { - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let freshness = self.check_path_modifications(RUSTC_IF_UNCHANGED_ALLOWED_PATHS); - self.verbose(|| { - eprintln!("rustc freshness: {freshness:?}"); - }); - match freshness { - PathFreshness::LastModifiedUpstream { upstream } => upstream, - PathFreshness::HasLocalModifications { upstream } => { - if if_unchanged { - return None; - } - - if self.is_running_on_ci { - eprintln!("CI rustc commit matches with HEAD and we are in CI."); - eprintln!( - "`rustc.download-ci` functionality will be skipped as artifacts are not available." - ); - return None; - } - - upstream - } - PathFreshness::MissingUpstream => { - eprintln!("No upstream commit found"); - return None; - } - } - } else { - channel::read_commit_info_file(&self.src) - .map(|info| info.sha.trim().to_owned()) - .expect("git-commit-info is missing in the project root") - }; - - if debug_assertions_requested { - eprintln!( - "WARN: `rust.debug-assertions = true` will prevent downloading CI rustc as alt CI \ - rustc is not currently built with debug assertions." - ); - return None; - } - - Some(commit) - } - - pub fn parse_download_ci_llvm( - &self, - download_ci_llvm: Option, - asserts: bool, - ) -> bool { - // We don't ever want to use `true` on CI, as we should not - // download upstream artifacts if there are any local modifications. - let default = if self.is_running_on_ci { - StringOrBool::String("if-unchanged".to_string()) - } else { - StringOrBool::Bool(true) - }; - let download_ci_llvm = download_ci_llvm.unwrap_or(default); - - let if_unchanged = || { - if self.rust_info.is_from_tarball() { - // Git is needed for running "if-unchanged" logic. - println!("ERROR: 'if-unchanged' is only compatible with Git managed sources."); - crate::exit!(1); - } - - // Fetching the LLVM submodule is unnecessary for self-tests. - #[cfg(not(test))] - self.update_submodule("src/llvm-project"); - - // Check for untracked changes in `src/llvm-project` and other important places. - let has_changes = self.has_changes_from_upstream(LLVM_INVALIDATION_PATHS); - - // Return false if there are untracked changes, otherwise check if CI LLVM is available. - if has_changes { false } else { llvm::is_ci_llvm_available_for_target(self, asserts) } - }; - - match download_ci_llvm { - StringOrBool::Bool(b) => { - if !b && self.download_rustc_commit.is_some() { - panic!( - "`llvm.download-ci-llvm` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`." - ); - } - - if b && self.is_running_on_ci { - // On CI, we must always rebuild LLVM if there were any modifications to it - panic!( - "`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead." - ); - } - - // If download-ci-llvm=true we also want to check that CI llvm is available - b && llvm::is_ci_llvm_available_for_target(self, asserts) - } - StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(), - StringOrBool::String(other) => { - panic!("unrecognized option for download-ci-llvm: {other:?}") - } - } - } - /// Returns true if any of the `paths` have been modified locally. pub fn has_changes_from_upstream(&self, paths: &[&'static str]) -> bool { match self.check_path_modifications(paths) { diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index 50eba12aba747..3ad9e528845ca 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -9,14 +9,15 @@ use build_helper::git::PathFreshness; use clap::CommandFactory; use serde::Deserialize; +use super::Config; use super::flags::Flags; use super::toml::change_id::ChangeIdWrapper; -use super::{Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS}; use crate::ChangeId; use crate::core::build_steps::clippy::{LintConfig, get_clippy_rules_in_order}; use crate::core::build_steps::llvm; use crate::core::build_steps::llvm::LLVM_INVALIDATION_PATHS; use crate::core::config::toml::TomlConfig; +use crate::core::config::toml::rust::RUSTC_IF_UNCHANGED_ALLOWED_PATHS; use crate::core::config::{LldMode, Target, TargetSelection}; use crate::utils::tests::git::git_test; diff --git a/src/bootstrap/src/core/config/toml/build.rs b/src/bootstrap/src/core/config/toml/build.rs index 4d29691f38b66..72dd7677f93d6 100644 --- a/src/bootstrap/src/core/config/toml/build.rs +++ b/src/bootstrap/src/core/config/toml/build.rs @@ -7,12 +7,16 @@ //! unless specifically overridden by other configuration sections or command-line flags. use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; use serde::{Deserialize, Deserializer}; +use crate::core::config::target_selection::TargetSelectionList; use crate::core::config::toml::ReplaceOpt; -use crate::core::config::{Merge, StringOrBool}; -use crate::{HashSet, PathBuf, define_config, exit}; +use crate::core::config::{Merge, StringOrBool, TargetSelection, set, threads_from_config}; +use crate::helpers::exe; +use crate::{Config, HashSet, PathBuf, Subcommand, command, define_config, exit, t}; define_config! { /// TOML representation of various global build decisions. @@ -82,3 +86,389 @@ define_config! { features: Option> = "features", } } + +impl Config { + pub fn apply_build_config( + &mut self, + toml_build: Option, + flags_skip_stage0_validation: bool, + flags_stage: Option, + flags_host: Option, + flags_target: Option, + flags_build_dir: Option, + ) { + let Build { + description, + build: _, + host, + target, + build_dir, + cargo, + rustc, + rustfmt, + cargo_clippy, + docs, + compiler_docs, + library_docs_private_items, + docs_minification, + submodules, + gdb, + lldb, + nodejs, + npm, + python, + reuse, + locked_deps, + vendor, + full_bootstrap, + bootstrap_cache_path, + extended, + tools, + tool, + verbose, + sanitizers, + profiler, + cargo_native_static, + low_priority, + configure_args, + local_rebuild, + print_step_timings, + print_step_rusage, + check_stage, + doc_stage, + build_stage, + test_stage, + install_stage, + dist_stage, + bench_stage, + patch_binaries_for_nix, + // This field is only used by bootstrap.py + metrics: _, + android_ndk, + optimized_compiler_builtins, + jobs, + compiletest_diff_tool, + compiletest_use_stage0_libtest, + tidy_extra_checks, + ccache, + exclude, + } = toml_build.unwrap_or_default(); + + if let Some(exclude) = exclude { + self.skip.extend( + exclude + .into_iter() + .map(|p| { + // Never return top-level path here as it would break `--skip` + // logic on rustc's internal test framework which is utilized + // by compiletest. + if cfg!(windows) { + PathBuf::from(p.to_str().unwrap().replace('/', "\\")) + } else { + p + } + }) + .collect::>(), + ); + } + + self.description = description; + match ccache { + Some(StringOrBool::String(ref s)) => self.ccache = Some(s.to_string()), + Some(StringOrBool::Bool(true)) => { + self.ccache = Some("ccache".to_string()); + } + Some(StringOrBool::Bool(false)) | None => {} + } + + if self.jobs.is_none() { + self.jobs = Some(threads_from_config(jobs.unwrap_or(0))); + } + + set(&mut self.out, flags_build_dir.or_else(|| build_dir.map(PathBuf::from))); + + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !self.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + self.out = std::path::absolute(&self.out).expect("can't make empty path absolute"); + } + + if cargo_clippy.is_some() && rustc.is_none() { + println!( + "WARNING: Using `build.cargo-clippy` without `build.rustc` usually fails due to toolchain conflict." + ); + } + + self.initial_rustc = if let Some(rustc) = rustc { + if !flags_skip_stage0_validation { + self.check_stage0_version(&rustc, "rustc"); + } + rustc + } else { + self.download_beta_toolchain(); + self.out + .join(self.host_target) + .join("stage0") + .join("bin") + .join(exe("rustc", self.host_target)) + }; + + self.initial_sysroot = t!(PathBuf::from_str( + command(&self.initial_rustc) + .args(["--print", "sysroot"]) + .run_in_dry_run() + .run_capture_stdout(&self) + .stdout() + .trim() + )); + + self.initial_cargo_clippy = cargo_clippy; + + self.initial_cargo = if let Some(cargo) = cargo { + if !flags_skip_stage0_validation { + self.check_stage0_version(&cargo, "cargo"); + } + cargo + } else { + self.download_beta_toolchain(); + self.initial_sysroot.join("bin").join(exe("cargo", self.host_target)) + }; + + // NOTE: it's important this comes *after* we set `initial_rustc` just above. + if self.dry_run() { + let dir = self.out.join("tmp-dry-run"); + t!(std::fs::create_dir_all(&dir)); + self.out = dir; + } + + self.hosts = flags_host + .map(|TargetSelectionList(list)| list) + .or_else(|| { + host.map(|h| h.into_iter().map(|s| TargetSelection::from_user(&s)).collect()) + }) + .unwrap_or_else(|| vec![self.host_target.clone()]); + + self.targets = flags_target + .map(|TargetSelectionList(list)| list) + .or_else(|| { + target.map(|t| t.into_iter().map(|s| TargetSelection::from_user(&s)).collect()) + }) + .unwrap_or_else(|| self.hosts.clone()); + + self.nodejs = nodejs.map(PathBuf::from); + self.npm = npm.map(PathBuf::from); + self.gdb = gdb.map(PathBuf::from); + self.lldb = lldb.map(PathBuf::from); + self.python = python.map(PathBuf::from); + self.reuse = reuse.map(PathBuf::from); + + self.submodules = submodules; + self.android_ndk = android_ndk; + self.bootstrap_cache_path = bootstrap_cache_path; + self.tools = tools; + + set(&mut self.low_priority, low_priority); + set(&mut self.compiler_docs, compiler_docs); + set(&mut self.library_docs_private_items, library_docs_private_items); + set(&mut self.docs_minification, docs_minification); + set(&mut self.docs, docs); + set(&mut self.locked_deps, locked_deps); + set(&mut self.full_bootstrap, full_bootstrap); + set(&mut self.extended, extended); + set(&mut self.tool, tool); + set(&mut self.verbose, verbose); + set(&mut self.sanitizers, sanitizers); + set(&mut self.profiler, profiler); + set(&mut self.cargo_native_static, cargo_native_static); + set(&mut self.configure_args, configure_args); + set(&mut self.local_rebuild, local_rebuild); + set(&mut self.print_step_timings, print_step_timings); + set(&mut self.print_step_rusage, print_step_rusage); + self.patch_binaries_for_nix = patch_binaries_for_nix; + + self.vendor = vendor.unwrap_or( + self.rust_info.is_from_tarball() + && self.src.join("vendor").exists() + && self.src.join(".cargo/config.toml").exists(), + ); + + self.initial_rustfmt = rustfmt.or_else(|| self.maybe_download_rustfmt()); + + self.optimized_compiler_builtins = + optimized_compiler_builtins.unwrap_or(self.channel != "dev"); + self.compiletest_diff_tool = compiletest_diff_tool; + self.compiletest_use_stage0_libtest = compiletest_use_stage0_libtest.unwrap_or(true); + self.tidy_extra_checks = tidy_extra_checks; + + let download_rustc = self.download_rustc_commit.is_some(); + self.explicit_stage_from_config = test_stage.is_some() + || build_stage.is_some() + || doc_stage.is_some() + || dist_stage.is_some() + || install_stage.is_some() + || check_stage.is_some() + || bench_stage.is_some(); + + self.stage = match self.cmd { + Subcommand::Check { .. } | Subcommand::Clippy { .. } | Subcommand::Fix => { + flags_stage.or(check_stage).unwrap_or(1) + } + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags_stage.or(doc_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Build => { + flags_stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } | Subcommand::Miri { .. } => { + flags_stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags_stage.or(bench_stage).unwrap_or(2), + Subcommand::Dist => flags_stage.or(dist_stage).unwrap_or(2), + Subcommand::Install => flags_stage.or(install_stage).unwrap_or(2), + Subcommand::Perf { .. } => flags_stage.unwrap_or(1), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Vendor { .. } => flags_stage.unwrap_or(0), + }; + } + + #[cfg(test)] + pub fn check_stage0_version(&self, _program_path: &Path, _component_name: &'static str) { + use std::path::Path; + } + + /// check rustc/cargo version is same or lower with 1 apart from the building one + #[cfg(not(test))] + pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { + use std::fs; + + use build_helper::util::fail; + + if self.dry_run() { + return; + } + + let stage0_output = + command(program_path).arg("--version").run_capture_stdout(self).stdout(); + let mut stage0_output = stage0_output.lines().next().unwrap().split(' '); + + let stage0_name = stage0_output.next().unwrap(); + if stage0_name != component_name { + fail(&format!( + "Expected to find {component_name} at {} but it claims to be {stage0_name}", + program_path.display() + )); + } + + let stage0_version = + semver::Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) + .unwrap(); + let source_version = semver::Version::parse( + fs::read_to_string(self.src.join("src/version")).unwrap().trim(), + ) + .unwrap(); + if !(source_version == stage0_version + || (source_version.major == stage0_version.major + && (source_version.minor == stage0_version.minor + || source_version.minor == stage0_version.minor + 1))) + { + let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); + fail(&format!( + "Unexpected {component_name} version: {stage0_version}, we should use {prev_version}/{source_version} to build source with {source_version}" + )); + } + } + + #[cfg(test)] + pub(crate) fn download_beta_toolchain(&self) {} + + #[cfg(not(test))] + pub(crate) fn download_beta_toolchain(&self) { + use crate::core::download::DownloadSource; + + self.verbose(|| println!("downloading stage0 beta artifacts")); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + #[cfg(test)] + pub(crate) fn maybe_download_rustfmt(&self) -> Option { + Some(PathBuf::new()) + } + + /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't + /// reuse target directories or artifacts + #[cfg(not(test))] + pub(crate) fn maybe_download_rustfmt(&self) -> Option { + use build_helper::stage0_parser::VersionMetadata; + + use crate::core::download::{DownloadSource, path_is_dylib}; + use crate::fs; + use crate::utils::build_stamp::BuildStamp; + + if self.dry_run() { + return Some(PathBuf::new()); + } + + let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = self.host_target; + let bin_root = self.out.join(host).join("rustfmt"); + let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); + let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel); + if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() { + return Some(rustfmt_path); + } + + self.download_component( + DownloadSource::Dist, + format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), + "rustfmt-preview", + date, + "rustfmt", + ); + self.download_component( + DownloadSource::Dist, + format!("rustc-{version}-{build}.tar.xz", build = host.triple), + "rustc", + date, + "rustfmt", + ); + + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if path_is_dylib(&lib.path()) { + self.fix_bin_or_dylib(&lib.path()); + } + } + } + + t!(rustfmt_stamp.write()); + Some(rustfmt_path) + } +} diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 1f0cecd145c77..b39541a4dd5d2 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Deserializer}; +use crate::core::build_steps::llvm::{self, LLVM_INVALIDATION_PATHS}; use crate::core::config::toml::{Merge, ReplaceOpt, TomlConfig}; use crate::core::config::{StringOrBool, set}; use crate::{Config, HashMap, HashSet, PathBuf, define_config, exit}; @@ -147,14 +148,9 @@ pub fn check_incompatible_options_for_ci_llvm( impl Config { pub fn apply_llvm_config(&mut self, toml_llvm: Option) { - let mut llvm_tests = None; - let mut llvm_enzyme = None; - let mut llvm_offload = None; - let mut llvm_plugins = None; - if let Some(llvm) = toml_llvm { let Llvm { - optimize: optimize_toml, + optimize, thin_lto, release_debuginfo, assertions: _, @@ -185,35 +181,36 @@ impl Config { } = llvm; set(&mut self.ninja_in_file, ninja); - llvm_tests = tests; - llvm_enzyme = enzyme; - llvm_offload = offload; - llvm_plugins = plugins; - set(&mut self.llvm_optimize, optimize_toml); + set(&mut self.llvm_optimize, optimize); set(&mut self.llvm_thin_lto, thin_lto); set(&mut self.llvm_release_debuginfo, release_debuginfo); set(&mut self.llvm_static_stdcpp, static_libstdcpp); set(&mut self.llvm_libzstd, libzstd); - if let Some(v) = link_shared { - self.llvm_link_shared.set(Some(v)); + set(&mut self.llvm_use_libcxx, use_libcxx); + + if let Some(shared) = link_shared { + self.llvm_link_shared.set(Some(shared)); } + self.llvm_targets.clone_from(&targets); self.llvm_experimental_targets.clone_from(&experimental_targets); self.llvm_link_jobs = link_jobs; self.llvm_version_suffix.clone_from(&version_suffix); self.llvm_clang_cl.clone_from(&clang_cl); - self.llvm_cflags.clone_from(&cflags); self.llvm_cxxflags.clone_from(&cxxflags); self.llvm_ldflags.clone_from(&ldflags); - set(&mut self.llvm_use_libcxx, use_libcxx); self.llvm_use_linker.clone_from(&use_linker); + self.llvm_build_config = build_config.unwrap_or_default(); + self.llvm_allow_old_toolchain = allow_old_toolchain.unwrap_or(false); self.llvm_offload = offload.unwrap_or(false); + self.llvm_tests = tests.unwrap_or(false); + self.llvm_enzyme = enzyme.unwrap_or(false); + self.llvm_plugins = plugins.unwrap_or(false); self.llvm_polly = polly.unwrap_or(false); self.llvm_clang = clang.unwrap_or(false); self.llvm_enable_warnings = enable_warnings.unwrap_or(false); - self.llvm_build_config = build_config.clone().unwrap_or(Default::default()); self.llvm_from_ci = self.parse_download_ci_llvm(download_ci_llvm, self.llvm_assertions); @@ -261,10 +258,66 @@ impl Config { } else { self.llvm_from_ci = self.parse_download_ci_llvm(None, false); } + } + + pub fn parse_download_ci_llvm( + &self, + download_ci_llvm: Option, + asserts: bool, + ) -> bool { + // We don't ever want to use `true` on CI, as we should not + // download upstream artifacts if there are any local modifications. + let default = if self.is_running_on_ci { + StringOrBool::String("if-unchanged".to_string()) + } else { + StringOrBool::Bool(true) + }; + let download_ci_llvm = download_ci_llvm.unwrap_or(default); + + let if_unchanged = || { + if self.rust_info.is_from_tarball() { + // Git is needed for running "if-unchanged" logic. + println!("ERROR: 'if-unchanged' is only compatible with Git managed sources."); + crate::exit!(1); + } + + // Fetching the LLVM submodule is unnecessary for self-tests. + #[cfg(not(test))] + self.update_submodule("src/llvm-project"); - self.llvm_tests = llvm_tests.unwrap_or(false); - self.llvm_enzyme = llvm_enzyme.unwrap_or(false); - self.llvm_offload = llvm_offload.unwrap_or(false); - self.llvm_plugins = llvm_plugins.unwrap_or(false); + // Check for untracked changes in `src/llvm-project` and other important places. + let has_changes = self.has_changes_from_upstream(LLVM_INVALIDATION_PATHS); + + // Return false if there are untracked changes, otherwise check if CI LLVM is available. + if has_changes { + false + } else { + crate::core::build_steps::llvm::is_ci_llvm_available_for_target(self, asserts) + } + }; + + match download_ci_llvm { + StringOrBool::Bool(b) => { + if !b && self.download_rustc_commit.is_some() { + panic!( + "`llvm.download-ci-llvm` cannot be set to `false` if `rust.download-rustc` is set to `true` or `if-unchanged`." + ); + } + + if b && self.is_running_on_ci { + // On CI, we must always rebuild LLVM if there were any modifications to it + panic!( + "`llvm.download-ci-llvm` cannot be set to `true` on CI. Use `if-unchanged` instead." + ); + } + + // If download-ci-llvm=true we also want to check that CI llvm is available + b && llvm::is_ci_llvm_available_for_target(self, asserts) + } + StringOrBool::String(s) if s == "if-unchanged" => if_unchanged(), + StringOrBool::String(other) => { + panic!("unrecognized option for download-ci-llvm: {other:?}") + } + } } } diff --git a/src/bootstrap/src/core/config/toml/rust.rs b/src/bootstrap/src/core/config/toml/rust.rs index c136bd4a6f9bd..ff7e374a26784 100644 --- a/src/bootstrap/src/core/config/toml/rust.rs +++ b/src/bootstrap/src/core/config/toml/rust.rs @@ -3,6 +3,7 @@ use std::str::FromStr; +use build_helper::git::PathFreshness; use serde::{Deserialize, Deserializer}; use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; @@ -11,8 +12,30 @@ use crate::core::config::{ DebuginfoLevel, Merge, ReplaceOpt, RustcLto, StringOrBool, set, threads_from_config, }; use crate::flags::Warnings; +use crate::utils::channel; use crate::{BTreeSet, Config, HashSet, PathBuf, TargetSelection, define_config, exit}; +/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. +/// This means they can be modified and changes to these paths should never trigger a compiler build +/// when "if-unchanged" is set. +/// +/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during +/// the diff check. +/// +/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build +/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. +/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the +/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. +#[rustfmt::skip] // We don't want rustfmt to oneline this list +pub const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ + ":!library", + ":!src/tools", + ":!src/librustdoc", + ":!src/rustdoc-json-types", + ":!tests", + ":!triagebot.toml", +]; + define_config! { /// TOML representation of how the Rust build is configured. struct Rust { @@ -228,6 +251,46 @@ impl RustOptimize { } } +/// Checks whether the CI rustc is available for the given target triple. +pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool { + // All tier 1 targets and tier 2 targets with host tools. + const SUPPORTED_PLATFORMS: &[&str] = &[ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "loongarch64-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", + ]; + + const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] = + &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"]; + + if llvm_assertions { + SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple) + } else { + SUPPORTED_PLATFORMS.contains(&target_triple) + } +} + /// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. /// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. pub fn check_incompatible_options_for_ci_rustc( @@ -500,7 +563,7 @@ impl Config { new_symbol_mangling, profile_generate, profile_use, - download_rustc, + download_rustc: _, lto, validate_mir_opts, frame_pointers, @@ -510,37 +573,6 @@ impl Config { std_features: std_features_toml, } = rust; - // FIXME(#133381): alt rustc builds currently do *not* have rustc debug assertions - // enabled. We should not download a CI alt rustc if we need rustc to have debug - // assertions (e.g. for crashes test suite). This can be changed once something like - // [Enable debug assertions on alt - // builds](https://github.com/rust-lang/rust/pull/131077) lands. - // - // Note that `rust.debug = true` currently implies `rust.debug-assertions = true`! - // - // This relies also on the fact that the global default for `download-rustc` will be - // `false` if it's not explicitly set. - let debug_assertions_requested = matches!(rustc_debug_assertions_toml, Some(true)) - || (matches!(debug_toml, Some(true)) - && !matches!(rustc_debug_assertions_toml, Some(false))); - - if debug_assertions_requested - && let Some(ref opt) = download_rustc - && opt.is_string_or_true() - { - eprintln!( - "WARN: currently no CI rustc builds have rustc debug assertions \ - enabled. Please either set `rust.debug-assertions` to `false` if you \ - want to use download CI rustc or set `rust.download-rustc` to `false`." - ); - } - - self.download_rustc_commit = self.download_ci_rustc_commit( - download_rustc, - debug_assertions_requested, - self.llvm_assertions, - ); - debug = debug_toml; rustc_debug_assertions = rustc_debug_assertions_toml; std_debug_assertions = std_debug_assertions_toml; @@ -656,8 +688,8 @@ impl Config { set(&mut self.lld_enabled, lld_enabled); } - let default_std_features = BTreeSet::from([String::from("panic-unwind")]); - self.rust_std_features = std_features.unwrap_or(default_std_features); + self.rust_std_features = + std_features.unwrap_or_else(|| BTreeSet::from([String::from("panic-unwind")])); let default = debug == Some(true); self.rustc_debug_assertions = rustc_debug_assertions.unwrap_or(default); @@ -680,4 +712,85 @@ impl Config { self.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); self.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); } + + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. + pub fn download_ci_rustc_commit( + &self, + download_rustc: Option, + debug_assertions_requested: bool, + llvm_assertions: bool, + ) -> Option { + if !is_download_ci_available(&self.host_target.triple, llvm_assertions) { + return None; + } + + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + // Globally default `download-rustc` to `false`, because some contributors don't use + // profiles for reasons such as: + // - They need to seamlessly switch between compiler/library work. + // - They don't want to use compiler profile because they need to override too many + // things and it's easier to not use a profile. + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => { + if !self.rust_info.is_managed_git_subrepository() { + println!( + "ERROR: `download-rustc=if-unchanged` is only compatible with Git managed sources." + ); + crate::exit!(1); + } + + true + } + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {other}") + } + }; + + let commit = if self.rust_info.is_managed_git_subrepository() { + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let freshness = self.check_path_modifications(RUSTC_IF_UNCHANGED_ALLOWED_PATHS); + self.verbose(|| { + eprintln!("rustc freshness: {freshness:?}"); + }); + match freshness { + PathFreshness::LastModifiedUpstream { upstream } => upstream, + PathFreshness::HasLocalModifications { upstream } => { + if if_unchanged { + return None; + } + + if self.is_running_on_ci { + eprintln!("CI rustc commit matches with HEAD and we are in CI."); + eprintln!( + "`rustc.download-ci` functionality will be skipped as artifacts are not available." + ); + return None; + } + + upstream + } + PathFreshness::MissingUpstream => { + eprintln!("No upstream commit found"); + return None; + } + } + } else { + channel::read_commit_info_file(&self.src) + .map(|info| info.sha.trim().to_owned()) + .expect("git-commit-info is missing in the project root") + }; + + if debug_assertions_requested { + eprintln!( + "WARN: `rust.debug-assertions = true` will prevent downloading CI rustc as alt CI \ + rustc is not currently built with debug assertions." + ); + return None; + } + + Some(commit) + } } diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index d7c6d8dbcc3f9..8cdd22229d4c3 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -67,7 +67,7 @@ impl Config { /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true /// on NixOS - fn should_fix_bins_and_dylibs(&self) -> bool { + pub(crate) fn should_fix_bins_and_dylibs(&self) -> bool { let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { let uname = command("uname").allow_failure().arg("-s").run_capture_stdout(self); if uname.is_failure() { @@ -120,7 +120,7 @@ impl Config { /// change the interpreter/RPATH of ELF executables. /// /// Please see for more information - fn fix_bin_or_dylib(&self, fname: &Path) { + pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) { assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); println!("attempting to patch {}", fname.display()); @@ -388,7 +388,7 @@ fn recorded_entries(dst: &Path, pattern: &str) -> Option> { Some(BufWriter::new(t!(File::create(dst.join(name))))) } -enum DownloadSource { +pub(crate) enum DownloadSource { CI, Dist, } @@ -420,63 +420,6 @@ impl Config { cargo_clippy } - #[cfg(test)] - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - Some(PathBuf::new()) - } - - /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't - /// reuse target directories or artifacts - #[cfg(not(test))] - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - use build_helper::stage0_parser::VersionMetadata; - - if self.dry_run() { - return Some(PathBuf::new()); - } - - let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; - let channel = format!("{version}-{date}"); - - let host = self.host_target; - let bin_root = self.out.join(host).join("rustfmt"); - let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel); - if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() { - return Some(rustfmt_path); - } - - self.download_component( - DownloadSource::Dist, - format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), - "rustfmt-preview", - date, - "rustfmt", - ); - self.download_component( - DownloadSource::Dist, - format!("rustc-{version}-{build}.tar.xz", build = host.triple), - "rustc", - date, - "rustfmt", - ); - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if path_is_dylib(&lib.path()) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - t!(rustfmt_stamp.write()); - Some(rustfmt_path) - } - pub(crate) fn ci_rust_std_contents(&self) -> Vec { self.ci_component_contents(".rust-std-contents") } @@ -514,31 +457,7 @@ impl Config { ); } - #[cfg(test)] - pub(crate) fn download_beta_toolchain(&self) {} - - #[cfg(not(test))] - pub(crate) fn download_beta_toolchain(&self) { - self.verbose(|| println!("downloading stage0 beta artifacts")); - - let date = &self.stage0_metadata.compiler.date; - let version = &self.stage0_metadata.compiler.version; - let extra_components = ["cargo"]; - - let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { - config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") - }; - - self.download_toolchain( - version, - "stage0", - date, - &extra_components, - download_beta_component, - ); - } - - fn download_toolchain( + pub(crate) fn download_toolchain( &self, version: &str, sysroot: &str, @@ -599,7 +518,7 @@ impl Config { ) } - fn download_component( + pub(crate) fn download_component( &self, mode: DownloadSource, filename: String, @@ -850,49 +769,50 @@ download-rustc = false } self.unpack(&tarball, root_dir, "gcc"); } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + pub(crate) fn artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = if self.rust_info.is_managed_git_subrepository() { + let channel = + self.read_file_by_commit(Path::new("src/ci/channel"), commit).trim().to_owned(); + let version = + self.read_file_by_commit(Path::new("src/version"), commit).trim().to_owned(); + (channel, version) + } else { + let channel = fs::read_to_string(self.src.join("src/ci/channel")); + let version = fs::read_to_string(self.src.join("src/version")); + match (channel, version) { + (Ok(channel), Ok(version)) => { + (channel.trim().to_owned(), version.trim().to_owned()) + } + (channel, version) => { + let src = self.src.display(); + eprintln!("ERROR: failed to determine artifact channel and/or version"); + eprintln!( + "HELP: consider using a git checkout or ensure these files are readable" + ); + if let Err(channel) = channel { + eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); + } + if let Err(version) = version { + eprintln!("reading {src}/src/version failed: {version:?}"); + } + panic!(); + } + } + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } } -fn path_is_dylib(path: &Path) -> bool { +pub(crate) fn path_is_dylib(path: &Path) -> bool { // The .so is not necessarily the extension, it might be libLLVM.so.18.1 path.to_str().is_some_and(|path| path.contains(".so")) } - -/// Checks whether the CI rustc is available for the given target triple. -pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool { - // All tier 1 targets and tier 2 targets with host tools. - const SUPPORTED_PLATFORMS: &[&str] = &[ - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "loongarch64-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ]; - - const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] = - &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"]; - - if llvm_assertions { - SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple) - } else { - SUPPORTED_PLATFORMS.contains(&target_triple) - } -} From 121be5a1ec8a1c6f48964a2c0f34ba8490215089 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 22 Jul 2025 16:41:00 +0530 Subject: [PATCH 2/2] move llvm_enzyme out of rust config, so that llvm_config initialization is preferred --- src/bootstrap/src/core/config/config.rs | 10 +++++++--- src/bootstrap/src/core/config/toml/build.rs | 2 +- src/bootstrap/src/core/config/toml/rust.rs | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 3b0652ff6c542..170c5e3000e25 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -718,7 +718,7 @@ impl Config { config.in_tree_llvm_info = config.git_info(false, &config.src.join("src/llvm-project")); config.in_tree_gcc_info = config.git_info(false, &config.src.join("src/gcc")); - toml.rust.as_ref().map(|rust| { + if let Some(rust) = toml.rust.as_ref() { // FIXME(#133381): alt rustc builds currently do *not* have rustc debug assertions // enabled. We should not download a CI alt rustc if we need rustc to have debug // assertions (e.g. for crashes test suite). This can be changed once something like @@ -749,7 +749,7 @@ impl Config { debug_assertions_requested, config.llvm_assertions, ); - }); + }; if !is_user_configured_rust_channel && config.rust_info.is_from_tarball() { config.channel = ci_channel.into(); @@ -795,9 +795,14 @@ impl Config { .collect::>(), ); + // Though we are applying it here, but the enzyme from llvm should get preference. + config.llvm_enzyme = + config.llvm_enzyme || config.channel == "dev" || config.channel == "nightly"; + config.apply_install_config(toml.install); config.apply_gcc_config(toml.gcc); config.apply_dist_config(toml.dist); + config.apply_llvm_config(toml.llvm); config.apply_build_config( toml.build, @@ -809,7 +814,6 @@ impl Config { ); config.apply_target_config(toml.target); config.apply_rust_config(toml.rust, flags_warnings); - config.apply_llvm_config(toml.llvm); if config.llvm_from_ci { let triple = &config.host_target.triple; diff --git a/src/bootstrap/src/core/config/toml/build.rs b/src/bootstrap/src/core/config/toml/build.rs index 72dd7677f93d6..a6fdddff58010 100644 --- a/src/bootstrap/src/core/config/toml/build.rs +++ b/src/bootstrap/src/core/config/toml/build.rs @@ -247,7 +247,7 @@ impl Config { .or_else(|| { host.map(|h| h.into_iter().map(|s| TargetSelection::from_user(&s)).collect()) }) - .unwrap_or_else(|| vec![self.host_target.clone()]); + .unwrap_or_else(|| vec![self.host_target]); self.targets = flags_target .map(|TargetSelectionList(list)| list) diff --git a/src/bootstrap/src/core/config/toml/rust.rs b/src/bootstrap/src/core/config/toml/rust.rs index ff7e374a26784..9fce49c2fb6d1 100644 --- a/src/bootstrap/src/core/config/toml/rust.rs +++ b/src/bootstrap/src/core/config/toml/rust.rs @@ -619,7 +619,6 @@ impl Config { self.rust_randomize_layout = randomize_layout.unwrap_or_default(); self.llvm_tools_enabled = llvm_tools.unwrap_or(true); - self.llvm_enzyme = self.channel == "dev" || self.channel == "nightly"; self.rustc_default_linker = default_linker; self.musl_root = musl_root.map(PathBuf::from); self.save_toolstates = save_toolstates.map(PathBuf::from);