From 3f6693b7e19a9abcf4bc851889d6fb260b04f27c Mon Sep 17 00:00:00 2001 From: Philipp Krones Date: Thu, 2 May 2024 17:30:30 +0200 Subject: [PATCH] Josh preparations --- Cargo.toml | 2 + clippy_config/Cargo.toml | 2 + clippy_dev/Cargo.toml | 3 + clippy_dev/src/lib.rs | 2 + clippy_dev/src/main.rs | 57 +++++++- clippy_dev/src/new_lint.rs | 15 ++- clippy_dev/src/release.rs | 76 +++++++++++ clippy_dev/src/sync.rs | 230 +++++++++++++++++++++++++++++++++ clippy_dev/src/update_lints.rs | 2 +- clippy_lints/Cargo.toml | 2 + clippy_utils/Cargo.toml | 2 + declare_clippy_lint/Cargo.toml | 2 + rust-toolchain | 2 + 13 files changed, 388 insertions(+), 9 deletions(-) create mode 100644 clippy_dev/src/release.rs create mode 100644 clippy_dev/src/sync.rs diff --git a/Cargo.toml b/Cargo.toml index b48f3ab3919c..fdcb92887399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "clippy" +# begin autogenerated version version = "0.1.80" +# end autogenerated version description = "A bunch of helpful lints to avoid common pitfalls in Rust" repository = "https://github.com/rust-lang/rust-clippy" readme = "README.md" diff --git a/clippy_config/Cargo.toml b/clippy_config/Cargo.toml index 7f7dc9d6cfb0..f63df52d68d2 100644 --- a/clippy_config/Cargo.toml +++ b/clippy_config/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "clippy_config" +# begin autogenerated version version = "0.1.80" +# end autogenerated version edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/clippy_dev/Cargo.toml b/clippy_dev/Cargo.toml index 4104e7d94f14..d644c1288e17 100644 --- a/clippy_dev/Cargo.toml +++ b/clippy_dev/Cargo.toml @@ -6,12 +6,15 @@ edition = "2021" [dependencies] aho-corasick = "1.0" +chrono = { version = "0.4.38", default-features = false, features = ["clock"] } clap = { version = "4.4", features = ["derive"] } +directories = "5" indoc = "1.0" itertools = "0.12" opener = "0.6" shell-escape = "0.1" walkdir = "2.3" +xshell = "0.2" [features] deny-warnings = [] diff --git a/clippy_dev/src/lib.rs b/clippy_dev/src/lib.rs index 385191e0361b..4608302809b9 100644 --- a/clippy_dev/src/lib.rs +++ b/clippy_dev/src/lib.rs @@ -24,8 +24,10 @@ pub mod dogfood; pub mod fmt; pub mod lint; pub mod new_lint; +pub mod release; pub mod serve; pub mod setup; +pub mod sync; pub mod update_lints; #[cfg(not(windows))] diff --git a/clippy_dev/src/main.rs b/clippy_dev/src/main.rs index 366b52b25dfc..8b690021f58f 100644 --- a/clippy_dev/src/main.rs +++ b/clippy_dev/src/main.rs @@ -3,7 +3,7 @@ #![warn(rust_2018_idioms, unused_lifetimes)] use clap::{Args, Parser, Subcommand}; -use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, update_lints}; +use clippy_dev::{dogfood, fmt, lint, new_lint, release, serve, setup, sync, update_lints}; use std::convert::Infallible; fn main() { @@ -75,6 +75,14 @@ fn main() { uplift, } => update_lints::rename(&old_name, new_name.as_ref().unwrap_or(&old_name), uplift), DevCommand::Deprecate { name, reason } => update_lints::deprecate(&name, reason.as_deref()), + DevCommand::Sync(SyncCommand { subcommand }) => match subcommand { + SyncSubcommand::Pull => sync::rustc_pull(), + SyncSubcommand::Push { repo_path, user, force } => sync::rustc_push(repo_path, &user, force), + }, + DevCommand::Release(ReleaseCommand { subcommand }) => match subcommand { + ReleaseSubcommand::BumpVersion => release::bump_version(), + ReleaseSubcommand::Commit { repo_path, branch } => release::rustc_clippy_commit(repo_path, branch), + }, } } @@ -225,6 +233,10 @@ enum DevCommand { /// The reason for deprecation reason: Option, }, + /// Sync between the rust repo and the Clippy repo + Sync(SyncCommand), + /// Manage Clippy releases + Release(ReleaseCommand), } #[derive(Args)] @@ -291,3 +303,46 @@ enum RemoveSubcommand { /// Remove the tasks added with 'cargo dev setup vscode-tasks' VscodeTasks, } + +#[derive(Args)] +struct SyncCommand { + #[command(subcommand)] + subcommand: SyncSubcommand, +} + +#[derive(Subcommand)] +enum SyncSubcommand { + /// Pull changes from rustc and update the toolchain + Pull, + /// Push changes to rustc + Push { + /// The path to a rustc repo that will be used for pushing changes + repo_path: String, + #[arg(long)] + /// The GitHub username to use for pushing changes + user: String, + #[arg(long, short)] + /// Force push changes + force: bool, + }, +} + +#[derive(Args)] +struct ReleaseCommand { + #[command(subcommand)] + subcommand: ReleaseSubcommand, +} + +#[derive(Subcommand)] +enum ReleaseSubcommand { + #[command(name = "bump_version")] + /// Bump the version in the Cargo.toml files + BumpVersion, + /// Print the Clippy commit in the rustc repo for the specified branch + Commit { + /// The path to a rustc repo to look for the commit + repo_path: String, + /// For which branch to print the commit + branch: release::Branch, + }, +} diff --git a/clippy_dev/src/new_lint.rs b/clippy_dev/src/new_lint.rs index b6481dde4dde..7b0f1a341ced 100644 --- a/clippy_dev/src/new_lint.rs +++ b/clippy_dev/src/new_lint.rs @@ -185,8 +185,8 @@ fn to_camel_case(name: &str) -> String { .collect() } -pub(crate) fn get_stabilization_version() -> String { - fn parse_manifest(contents: &str) -> Option { +pub(crate) fn clippy_version() -> (u32, u32) { + fn parse_manifest(contents: &str) -> Option<(u32, u32)> { let version = contents .lines() .filter_map(|l| l.split_once('=')) @@ -195,16 +195,17 @@ pub(crate) fn get_stabilization_version() -> String { return None; }; let (minor, patch) = version.split_once('.')?; - Some(format!( - "{}.{}.0", - minor.parse::().ok()?, - patch.parse::().ok()? - )) + Some((minor.parse().ok()?, patch.parse().ok()?)) } let contents = fs::read_to_string("Cargo.toml").expect("Unable to read `Cargo.toml`"); parse_manifest(&contents).expect("Unable to find package version in `Cargo.toml`") } +pub(crate) fn get_stabilization_version() -> String { + let (minor, patch) = clippy_version(); + format!("{minor}.{patch}.0") +} + fn get_test_file_contents(lint_name: &str, msrv: bool) -> String { let mut test = formatdoc!( r#" diff --git a/clippy_dev/src/release.rs b/clippy_dev/src/release.rs new file mode 100644 index 000000000000..8edf8f993a34 --- /dev/null +++ b/clippy_dev/src/release.rs @@ -0,0 +1,76 @@ +use std::fmt::{Display, Write}; +use std::path::Path; + +use crate::new_lint::clippy_version; +use crate::update_lints::{replace_region_in_file, UpdateMode}; + +use clap::ValueEnum; +use xshell::{cmd, Shell}; + +const CARGO_TOML_FILES: [&str; 5] = [ + "clippy_config/Cargo.toml", + "clippy_lints/Cargo.toml", + "clippy_utils/Cargo.toml", + "declare_clippy_lint/Cargo.toml", + "Cargo.toml", +]; + +pub fn bump_version() { + let (minor, mut patch) = clippy_version(); + patch += 1; + for file in &CARGO_TOML_FILES { + replace_region_in_file( + UpdateMode::Change, + Path::new(file), + "# begin autogenerated version\n", + "# end autogenerated version", + |res| { + writeln!(res, "version = \"0.{minor}.{patch}\"").unwrap(); + }, + ); + } +} + +#[derive(ValueEnum, Copy, Clone)] +pub enum Branch { + Stable, + Beta, + Master, +} + +impl Display for Branch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Branch::Stable => write!(f, "stable"), + Branch::Beta => write!(f, "beta"), + Branch::Master => write!(f, "master"), + } + } +} + +pub fn rustc_clippy_commit(rustc_path: String, branch: Branch) { + let sh = Shell::new().expect("failed to create shell"); + sh.change_dir(rustc_path); + + let base = branch.to_string(); + cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}") + .run() + .expect("failed to fetch base commit"); + let last_rustup_commit = cmd!( + sh, + "git log -1 --merges --grep=\"Sync from Clippy commit:\" FETCH_HEAD -- src/tools/clippy" + ) + .read() + .expect("failed to run git log"); + + let commit = last_rustup_commit + .lines() + .find(|c| c.contains("Sync from Clippy commit:")) + .expect("no commit found") + .trim() + .rsplit_once('@') + .expect("no commit hash found") + .1; + + println!("{commit}"); +} diff --git a/clippy_dev/src/sync.rs b/clippy_dev/src/sync.rs new file mode 100644 index 000000000000..3647ecceae14 --- /dev/null +++ b/clippy_dev/src/sync.rs @@ -0,0 +1,230 @@ +use std::fmt::Write; +use std::path::Path; +use std::process; +use std::process::exit; + +use chrono::offset::Utc; +use xshell::{cmd, Shell}; + +use crate::{clippy_project_root, update_lints}; + +const JOSH_FILTER: &str = ":rev(20b085d500dfba5afe0869707bf357af3afe20be:prefix=src/tools/clippy):/src/tools/clippy"; +const JOSH_PORT: &str = "42042"; + +fn start_josh() -> impl Drop { + // Create a wrapper that stops it on drop. + struct Josh(process::Child); + impl Drop for Josh { + fn drop(&mut self) { + #[cfg(unix)] + { + // Try to gracefully shut it down. + process::Command::new("kill") + .args(["-s", "INT", &self.0.id().to_string()]) + .output() + .expect("failed to SIGINT josh-proxy"); + // Sadly there is no "wait with timeout"... so we just give it some time to finish. + std::thread::sleep(std::time::Duration::from_secs(1)); + // Now hopefully it is gone. + if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() { + return; + } + } + // If that didn't work (or we're not on Unix), kill it hard. + eprintln!("I have to kill josh-proxy the hard way, let's hope this does not break anything."); + self.0.kill().expect("failed to SIGKILL josh-proxy"); + } + } + + // Determine cache directory. + let local_dir = { + let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "clippy-josh").unwrap(); + user_dirs.cache_dir().to_owned() + }; + println!("Using local cache directory: {}", local_dir.display()); + + // Start josh, silencing its output. + let mut cmd = process::Command::new("josh-proxy"); + cmd.arg("--local").arg(local_dir); + cmd.arg("--remote").arg("https://github.com"); + cmd.arg("--port").arg(JOSH_PORT); + cmd.arg("--no-background"); + cmd.stdout(process::Stdio::null()); + cmd.stderr(process::Stdio::null()); + let josh = cmd + .spawn() + .expect("failed to start josh-proxy, make sure it is installed"); + // Give it some time so hopefully the port is open. + std::thread::sleep(std::time::Duration::from_secs(1)); + + Josh(josh) +} + +fn rustc_hash() -> String { + std::env::remove_var("RUSTUP_TOOLCHAIN"); + let sh = Shell::new().expect("failed to create shell"); + cmd!(sh, "rustc --version --verbose") + .read() + .expect("failed to run `rustc -vV`") + .lines() + .find(|line| line.starts_with("commit-hash:")) + .expect("failed to parse `rustc -vV`") + .split_whitespace() + .last() + .expect("failed to get commit from `rustc -vV`") + .to_string() +} + +fn assert_clean_repo(sh: &Shell) { + if !cmd!(sh, "git status --untracked-files=no --porcelain") + .read() + .expect("failed to run git status") + .is_empty() + { + eprintln!("working directory must be clean before running `cargo dev sync pull`"); + exit(1); + } +} + +pub fn rustc_pull() { + const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc"; + + let sh = Shell::new().expect("failed to create shell"); + sh.change_dir(clippy_project_root()); + + assert_clean_repo(&sh); + + // Update rust-toolchain file + let date = Utc::now().format("%Y-%m-%d").to_string(); + update_lints::replace_region_in_file( + update_lints::UpdateMode::Change, + Path::new("rust-toolchain"), + "# begin autogenerated version\n", + "# end autogenerated version", + |res| { + writeln!(res, "channel = \"nightly-{date}\"").unwrap(); + }, + ); + + let message = format!("Bump nightly version -> {date}"); + cmd!(sh, "git commit rust-toolchain --no-verify -m {message}") + .run() + .expect("FAILED to commit rust-toolchain file, something went wrong"); + + let commit = rustc_hash(); + + // Make sure josh is running in this scope + { + let _josh = start_josh(); + + // Fetch given rustc commit. + cmd!( + sh, + "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git" + ) + .run() + .expect("FAILED to fetch new commits, something went wrong"); + } + + // This should not add any new root commits. So count those before and after merging. + let num_roots = || -> u32 { + cmd!(sh, "git rev-list HEAD --max-parents=0 --count") + .read() + .expect("failed to determine the number of root commits") + .parse::() + .unwrap() + }; + let num_roots_before = num_roots(); + + // Merge the fetched commit. + cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}") + .run() + .expect("FAILED to merge new commits, something went wrong"); + + // Check that the number of roots did not increase. + if num_roots() != num_roots_before { + eprintln!("Josh created a new root commit. This is probably not the history you want."); + exit(1); + } +} + +pub fn rustc_push(rustc_path: String, github_user: &str, force: bool) { + const BRANCH: &str = "clippy-subtree-update"; + + let sh = Shell::new().expect("failed to create shell"); + sh.change_dir(clippy_project_root()); + + assert_clean_repo(&sh); + + // Prepare the branch. Pushing works much better if we use as base exactly + // the commit that we pulled from last time, so we use the `rustc --version` + // to find out which commit that would be. + let base = rustc_hash(); + + println!("Preparing {github_user}/rust (base: {base})..."); + sh.change_dir(rustc_path); + if !force + && cmd!(sh, "git fetch https://github.com/{github_user}/rust {BRANCH}") + .ignore_stderr() + .read() + .is_ok() + { + eprintln!( + "The branch '{BRANCH}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again." + ); + exit(1); + } + cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}") + .run() + .expect("failed to fetch base commit"); + let force_flag = if force { "--force-with-lease" } else { "" }; + cmd!( + sh, + "git push https://github.com/{github_user}/rust {base}:refs/heads/{BRANCH} {force_flag}" + ) + .ignore_stdout() + .ignore_stderr() // silence the "create GitHub PR" message + .run() + .expect("failed to push base commit to the new branch"); + + // Make sure josh is running in this scope + { + let _josh = start_josh(); + + // Do the actual push. + sh.change_dir(clippy_project_root()); + println!("Pushing Clippy changes..."); + cmd!( + sh, + "git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{BRANCH}" + ) + .run() + .expect("failed to push changes to Josh"); + + // Do a round-trip check to make sure the push worked as expected. + cmd!( + sh, + "git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {BRANCH}" + ) + .ignore_stderr() + .read() + .expect("failed to fetch the branch from Josh"); + } + + let head = cmd!(sh, "git rev-parse HEAD") + .read() + .expect("failed to get HEAD commit"); + let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD") + .read() + .expect("failed to get FETCH_HEAD"); + if head != fetch_head { + eprintln!("Josh created a non-roundtrip push! Do NOT merge this into rustc!"); + exit(1); + } + println!("Confirmed that the push round-trips back to Clippy properly. Please create a rustc PR:"); + println!( + // Open PR with `subtree update` title to silence the `no-merges` triagebot check + // See https://github.com/rust-lang/rust/pull/114157 + " https://github.com/rust-lang/rust/compare/{github_user}:{BRANCH}?quick_pull=1&title=Clippy+subtree+update&body=r?+@ghost%0A%0ASync+from+Clippy+commit:+rust-lang%2Frust-clippy%40{head}" + ); +} diff --git a/clippy_dev/src/update_lints.rs b/clippy_dev/src/update_lints.rs index 45353901c98f..486fe09b01c6 100644 --- a/clippy_dev/src/update_lints.rs +++ b/clippy_dev/src/update_lints.rs @@ -946,7 +946,7 @@ fn remove_line_splices(s: &str) -> String { /// # Panics /// /// Panics if the path could not read or then written -fn replace_region_in_file( +pub(crate) fn replace_region_in_file( update_mode: UpdateMode, path: &Path, start: &str, diff --git a/clippy_lints/Cargo.toml b/clippy_lints/Cargo.toml index 5e3a119337cc..0c404256ac04 100644 --- a/clippy_lints/Cargo.toml +++ b/clippy_lints/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "clippy_lints" +# begin autogenerated version version = "0.1.80" +# end autogenerated version description = "A bunch of helpful lints to avoid common pitfalls in Rust" repository = "https://github.com/rust-lang/rust-clippy" readme = "README.md" diff --git a/clippy_utils/Cargo.toml b/clippy_utils/Cargo.toml index ab883c25338b..aa8cb3e93a32 100644 --- a/clippy_utils/Cargo.toml +++ b/clippy_utils/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "clippy_utils" +# begin autogenerated version version = "0.1.80" +# end autogenerated version edition = "2021" publish = false diff --git a/declare_clippy_lint/Cargo.toml b/declare_clippy_lint/Cargo.toml index c8c734c3a7c9..52eed2db9aec 100644 --- a/declare_clippy_lint/Cargo.toml +++ b/declare_clippy_lint/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "declare_clippy_lint" +# begin autogenerated version version = "0.1.80" +# end autogenerated version edition = "2021" publish = false diff --git a/rust-toolchain b/rust-toolchain index 055f305eb8e1..8e143f0e354a 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,5 @@ [toolchain] +# begin autogenerated version channel = "nightly-2024-05-02" +# end autogenerated version components = ["cargo", "llvm-tools", "rust-src", "rust-std", "rustc", "rustc-dev", "rustfmt"]