Rollup merge of #40060 - alexcrichton:retry-downloads, r=aturon

rustbuild: Retry downloads by default

Don't rely on curl's --retry, it appears to not work for some errors like SSL
errors.

Author: frewsxcv

src/bootstrap/bootstrap.py

@@ -59,20 +59,33 @@ def delete_if_present(path, verbose):
 
 
 def download(path, url, probably_big, verbose):
+    for x in range(0, 4):
+        try:
+            _download(path, url, probably_big, verbose, True)
+            return
+        except RuntimeError:
+            print("\nspurious failure, trying again")
+    _download(path, url, probably_big, verbose, False)
+
+
+def _download(path, url, probably_big, verbose, exception):
     if probably_big or verbose:
         print("downloading {}".format(url))
     # see http://serverfault.com/questions/301128/how-to-download
     if sys.platform == 'win32':
         run(["PowerShell.exe", "/nologo", "-Command",
              "(New-Object System.Net.WebClient)"
              ".DownloadFile('{}', '{}')".format(url, path)],
-            verbose=verbose)
+            verbose=verbose,
+            exception=exception)
     else:
         if probably_big or verbose:
             option = "-#"
         else:
             option = "-s"
-        run(["curl", option, "--retry", "3", "-Sf", "-o", path, url], verbose=verbose)
+        run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
+            verbose=verbose,
+            exception=exception)
 
 
 def verify(path, sha_path, verbose):
@@ -112,7 +125,7 @@ def unpack(tarball, dst, verbose=False, match=None):
             shutil.move(tp, fp)
     shutil.rmtree(os.path.join(dst, fname))
 
-def run(args, verbose=False):
+def run(args, verbose=False, exception=False):
     if verbose:
         print("running: " + ' '.join(args))
     sys.stdout.flush()
@@ -122,7 +135,7 @@ def run(args, verbose=False):
     code = ret.wait()
     if code != 0:
         err = "failed to run: " + ' '.join(args)
-        if verbose:
+        if verbose or exception:
             raise RuntimeError(err)
         sys.exit(err)