Auto merge of #96602 - TApplencourt:patch-1, r=Mark-Simulacrum

boostrap.py use curl by default

Fixes #61611

Author: bors

src/bootstrap/bootstrap.py

@@ -110,29 +110,42 @@ def download(path, url, probably_big, verbose, help_on_error=None):
 
 
 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
+    # Try to use curl (potentially available on win32
+    #    https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/)
+    # If an error occurs:
+    #  - If we are on win32 fallback to powershell
+    #  - Otherwise raise the error if appropriate
     if probably_big or verbose:
         print("downloading {}".format(url))
-    # see https://serverfault.com/questions/301128/how-to-download
-    if sys.platform == 'win32':
-        run(["PowerShell.exe", "/nologo", "-Command",
-             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
-             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
-            verbose=verbose,
-            exception=exception)
-    else:
+
+    platform_is_win32 = sys.platform == 'win32'
+    try:
         if probably_big or verbose:
             option = "-#"
         else:
             option = "-s"
-        require(["curl", "--version"])
+        # If curl is not present on Win32, we shoud not sys.exit
+        #   but raise `CalledProcessError` or `OSError` instead
+        require(["curl", "--version"], exception=platform_is_win32)
         run(["curl", option,
              "-L", # Follow redirect.
              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
              "--retry", "3", "-Sf", "-o", path, url],
             verbose=verbose,
-            exception=exception,
+            exception=True, # Will raise RuntimeError on failure
             help_on_error=help_on_error)
+    except (subprocess.CalledProcessError, OSError, RuntimeError):
+        # see http://serverfault.com/questions/301128/how-to-download
+        if platform_is_win32:
+            run(["PowerShell.exe", "/nologo", "-Command",
+                 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+                 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
+                verbose=verbose,
+                exception=exception)
+        # Check if the RuntimeError raised by run(curl) should be silenced
+        elif verbose or exception:
+            raise
 
 
 def verify(path, expected, verbose):
@@ -198,19 +211,23 @@ def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=
             sys.exit(err)
 
 
-def require(cmd, exit=True):
+def require(cmd, exit=True, exception=False):
     '''Run a command, returning its output.
     On error,
-        If `exit` is `True`, exit the process.
-        Otherwise, return None.'''
+        If `exception` is `True`, raise the error
+        Otherwise If `exit` is `True`, exit the process
+        Else return None.'''
     try:
         return subprocess.check_output(cmd).strip()
     except (subprocess.CalledProcessError, OSError) as exc:
-        if not exit:
-            return None
-        print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
-        print("Please make sure it's installed and in the path.")
-        sys.exit(1)
+        if exception:
+            raise
+        elif exit:
+            print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
+            print("Please make sure it's installed and in the path.")
+            sys.exit(1)
+        return None
+
 
 
 def format_build_time(duration):

src/bootstrap/lib.rs

@@ -122,7 +122,8 @@ use once_cell::sync::OnceCell;
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
 use crate::util::{
-    exe, libdir, mtime, output, run, run_suppressed, t, try_run, try_run_suppressed, CiEnv,
+    check_run, exe, libdir, mtime, output, run, run_suppressed, t, try_run, try_run_suppressed,
+    CiEnv,
 };
 
 mod builder;
@@ -961,6 +962,17 @@ impl Build {
         try_run_suppressed(cmd)
     }
 
+    /// Runs a command, printing out nice contextual information if it fails.
+    /// Returns false if do not execute at all, otherwise returns its
+    /// `status.success()`.
+    fn check_run(&self, cmd: &mut Command) -> bool {
+        if self.config.dry_run {
+            return true;
+        }
+        self.verbose(&format!("running: {:?}", cmd));
+        check_run(cmd, self.is_verbose())
+    }
+
     pub fn is_verbose(&self) -> bool {
         self.verbosity > 0
     }

src/bootstrap/native.rs

@@ -306,39 +306,40 @@ fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &
 
 fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) {
     println!("downloading {}", url);
-
-    // FIXME: check if curl is installed instead of skipping straight to powershell
-    if builder.build.build.contains("windows-msvc") {
-        for _ in 0..3 {
-            if builder.try_run(Command::new("PowerShell.exe").args(&[
-                "/nologo",
-                "-Command",
-                "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
-                &format!(
-                    "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
-                    url, tempfile
-                ),
-            ])) {
-                return;
+    // Try curl. If that fails and we are on windows, fallback to PowerShell.
+    if !builder.check_run(Command::new("curl").args(&[
+        "-#",
+        "-y",
+        "30",
+        "-Y",
+        "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
+        "--connect-timeout",
+        "30", // timeout if cannot connect within 30 seconds
+        "--retry",
+        "3",
+        "-Sf",
+        "-o",
+        tempfile,
+        url,
+    ])) {
+        if builder.build.build.contains("windows-msvc") {
+            println!("Fallback to PowerShell");
+            for _ in 0..3 {
+                if builder.try_run(Command::new("PowerShell.exe").args(&[
+                    "/nologo",
+                    "-Command",
+                    "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+                    &format!(
+                        "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
+                        url, tempfile
+                    ),
+                ])) {
+                    return;
+                }
+                println!("\nspurious failure, trying again");
             }
-            println!("\nspurious failure, trying again");
         }
-    } else {
-        builder.run(Command::new("curl").args(&[
-            "-#",
-            "-y",
-            "30",
-            "-Y",
-            "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
-            "--connect-timeout",
-            "30", // timeout if cannot connect within 30 seconds
-            "--retry",
-            "3",
-            "-Sf",
-            "-o",
-            tempfile,
-            url,
-        ]));
+        std::process::exit(1);
     }
 }
 

src/bootstrap/util.rs

@@ -346,6 +346,24 @@ pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool {
     status.success()
 }
 
+pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool {
+    let status = match cmd.status() {
+        Ok(status) => status,
+        Err(e) => {
+            println!("failed to execute command: {:?}\nerror: {}", cmd, e);
+            return false;
+        }
+    };
+    if !status.success() && print_cmd_on_fail {
+        println!(
+            "\n\ncommand did not execute successfully: {:?}\n\
+             expected success, got: {}\n\n",
+            cmd, status
+        );
+    }
+    status.success()
+}
+
 pub fn run_suppressed(cmd: &mut Command) {
     if !try_run_suppressed(cmd) {
         std::process::exit(1);