Retry Seed Download (Azure#23353)

* seed stage will attempt re-download. bump to newer version of virtualenv. bump to a newer version of pip in prep_environment
* ensure we install wheel so that we aren't forced onto setup.py only during installation of ci_tools.txt
* moving pin of black so we don't install it when it is unused

Co-authored-by: Ben Broderick Phillips <[email protected]>

Author: scbedd

eng/ci_tools.txt

@@ -2,7 +2,7 @@
 cryptography<4
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
-virtualenv==20.0.23
+virtualenv==20.13.2
 wheel==0.37.0
 Jinja2==2.11.2
 packaging==20.4
@@ -17,7 +17,6 @@ codecov==2.1.0
 beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
 pip==20.3.3
-black==21.6b0; python_version >= '3.6'
 wrapt<=1.12.1; python_version == '2.7'
 markupsafe==2.0.1; python_version > '2.7'
 markupsafe==1.1.1; python_version == '2.7'

eng/pipelines/templates/steps/build-test.yml

@@ -32,7 +32,8 @@ steps:
       versionSpec: '${{ parameters.PythonVersion }}'
 
   - script: |
-      python -m pip install pip==20.1
+      python -m pip install pip==20.3.3
+      python -m pip install wheel==0.37.0
       pip install -r eng/ci_tools.txt
       pip --version
       pip freeze

eng/pipelines/templates/steps/run_black.yml

@@ -11,7 +11,7 @@ steps:
     condition: succeededOrFailed()
 
   - script: |
-      pip install -r eng/ci_tools.txt
+      pip install black==21.6b0
     displayName: 'Prep Environment'
     condition: succeededOrFailed()
 

eng/pipelines/templates/steps/seed-virtualenv-wheels.yml

@@ -14,11 +14,9 @@ steps:
     displayName: Set VIRTUALENV_OVERRIDE_APP_DATA
 
   - pwsh: |
-      # ensure these can be pulled down from pypi.
-      $env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple"
-      virtualenv --download --reset-app-data `
-        --pip="${{ parameters.PIP_VERSION }}" `
-        --setuptools="${{ parameters.SETUPTOOLS_VERSION }}" `
-        --wheel=="${{ parameters.WHEEL_VERSION }}" `
-        $(Agent.TempDirectory)/seed_env
+      $(Build.SourcesDirectory)/eng/scripts/seed-virtualenv-wheels.ps1 `
+        -Pip "${{ parameters.PIP_VERSION }}" `
+        -SetupTools "${{ parameters.SETUPTOOLS_VERSION }}" `
+        -Wheel "${{ parameters.WHEEL_VERSION }}" `
+        -SeedDirectory "$(Agent.TempDirectory)/seed_env"
     displayName: Populate the seed dir

eng/scripts/seed-virtualenv-wheels.ps1

@@ -0,0 +1,49 @@
+<#
+.SYNOPSIS
+Creates a virtual environment while seeding with specific versions of setuptools, wheel, and pip.
+
+.DESCRIPTION
+Creates a virtual environment with three specifically targeted wheel versions. When used in conjunction with the environment variable 
+VIRTUALENV_OVERRIDE_APP_DATA, further virtualenv creations will leverage these targeted wheels specifically.
+
+.PARAMETER Pip
+The targeted pip version.
+
+.PARAMETER SetupTools
+The targeted setuptools version.
+
+.PARAMETER Wheel
+The targeted wheel version.
+
+.PARAMETER SeedDirectory
+The location of the virtualenv that will be created.
+#>
+param (
+  $Pip,
+  $SetupTools,
+  $Wheel,
+  $SeedDirectory
+)
+
+$attempts = 0
+
+# ensure these can be pulled down from pypi.
+$env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple"
+
+while ($attempts -lt 3) {
+  virtualenv --download --reset-app-data `
+    --pip="$Pip" `
+    --setuptools="$SetupTools" `
+    --wheel="$Wheel" `
+    "$SeedDirectory"
+  if ($LASTEXITCODE -eq 0) {
+    break
+  }
+
+  $attempts += 1
+}
+
+if ($LASTEXITCODE -ne 0) {
+  Write-Error "Unable to successfully populate an app data directory."
+  exit $LASTEXITCODE
+}