Sync eng/common directory with azure-sdk-tools for PR 2758 (Azure#23068)

* tooling updates. allow install but no run of test proxy, prepending the path properly, etc
* test updates for parallel invocation of test-proxy

Co-authored-by: scbedd <[email protected]>

Author: azure-sdk

.gitignore

@@ -44,6 +44,7 @@ pylint-*.out.txt
 coverage-*.xml
 stderr.txt
 stdout.txt
+_proxy_log_*.log
 
 # tox environment folders
 .tox/

.vscode/cspell.json

@@ -110,7 +110,9 @@
     "eng/*.txt",
     "eng/tox/tox.ini",
     "eng/common/docgeneration/Generate-DocIndex.ps1",
-    "eng/tox/*.py"
+    "eng/**/*.py",
+    ".gitignore",
+    
   ],
   "words": [
     "aad",
@@ -264,7 +266,9 @@
     "wfile",
     "westus",
     "docfx",
-    "Docfx"
+    "Docfx",
+    "azurecr",
+    "envname"
   ],
   "overrides": [
     {

doc/dev/test_proxy_troubleshooting.md

@@ -61,7 +61,7 @@ the wrong repo.
 
 Try updating your machine's version of Docker. Older versions of Docker may not return a status to indicate whether or
 not the proxy container is running, which the
-[proxy_docker_startup.py](https://github.com/Azure/azure-sdk-for-python/blob/main/tools/azure-sdk-tools/devtools_testutils/proxy_docker_startup.py)
+[proxy_startup.py](https://github.com/Azure/azure-sdk-for-python/blob/main/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py)
 script needs to determine.
 
 ## ConnectionError during test startup

eng/common/testproxy/docker-start-proxy.ps1

@@ -31,7 +31,7 @@ catch {
     Write-Error "Please check your docker invocation and try running the script again."
 }
 
-$SELECTED_IMAGE_TAG = "1314089"
+$SELECTED_IMAGE_TAG = "1369319"
 $CONTAINER_NAME = "ambitious_azsdk_test_proxy"
 $LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}"
 $WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}"

eng/common/testproxy/test-proxy-tool.yml

@@ -1,34 +1,40 @@
 parameters:
   rootFolder: '$(Build.SourcesDirectory)'
+  runProxy: true
 
 steps:
   - pwsh: |
         $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1
     displayName: 'Language Specific Certificate Trust'
 
-  - pwsh: |
-      Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx"
-      Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password"
-      Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true"
-    displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables'
-
   - pwsh: |
       dotnet tool install azure.sdk.tools.testproxy `
         --tool-path $(Build.BinariesDirectory)/test-proxy `
         --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json `
-        --version 1.0.0-dev.20220119.2
+        --version 1.0.0-dev.20220210.1
     displayName: "Install test-proxy"
 
   - pwsh: |
-      Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe `
-        -ArgumentList "--storage-location ${{ parameters.rootFolder }}" `
-        -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/test-proxy.log
-    displayName: 'Run the testproxy - windows'
-    condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'))
+      Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy"
+    displayName: "Prepend path with test-proxy tool install location"
+
+  - ${{ if eq(parameters.runProxy, 'true') }}:
+    - pwsh: |
+        Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx"
+        Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password"
+        Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true"
+      displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables'
+
+    - pwsh: |
+        Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe `
+          -ArgumentList "--storage-location ${{ parameters.rootFolder }}" `
+          -NoNewWindow -PassThru -RedirectStandardOutput $(Build.SourcesDirectory)/test-proxy.log
+      displayName: 'Run the testproxy - windows'
+      condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'))
 
-  # nohup does NOT continue beyond the current session if you use it within powershell
-  - bash: |
-      nohup $(Build.BinariesDirectory)/test-proxy/test-proxy > $(Build.SourcesDirectory)/test-proxy.log &
-    displayName: "Run the testproxy - linux/mac"
-    condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'))
-    workingDirectory: "${{ parameters.rootFolder }}"
+    # nohup does NOT continue beyond the current session if you use it within powershell
+    - bash: |
+        nohup $(Build.BinariesDirectory)/test-proxy/test-proxy > $(Build.SourcesDirectory)/test-proxy.log &
+      displayName: "Run the testproxy - linux/mac"
+      condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'))
+      workingDirectory: "${{ parameters.rootFolder }}"

eng/pipelines/templates/jobs/ci.conda.tests.yml

@@ -201,5 +201,10 @@ jobs:
 
     - ${{if eq(parameters.TestProxy, true) }}:
       - pwsh: |
-          cat $(Build.SourcesDirectory)/test-proxy.log
+          $files = Get-ChildItem -Path $(Build.SourcesDirectory) -Filter _proxy_log_*.log
+          foreach($file in $files){
+              Write-Host "##[group]$file"
+              cat $file
+              Write-Host "##[endgroup]"
+          }
         displayName: Dump Test-Proxy Logs

eng/pipelines/templates/steps/build-test.yml

@@ -40,6 +40,8 @@ steps:
 
   - ${{if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-tool.yml
+      parameters:
+        runProxy: false
 
   - ${{if eq(variables['System.TeamProject'], 'internal') }}:
     - template: ../steps/auth-dev-feed.yml
@@ -65,7 +67,12 @@ steps:
 
   - ${{if eq(parameters.TestProxy, true) }}:
     - pwsh: |
-        cat $(Build.SourcesDirectory)/test-proxy.log
+        $files = Get-ChildItem -Path $(Build.SourcesDirectory) -Filter _proxy_log_*.log
+        foreach($file in $files){
+            Write-Host "##[group]$file"
+            cat $file
+            Write-Host "##[endgroup]"
+        }
       displayName: Dump Test-Proxy Logs
       condition: succeededOrFailed()
 

eng/tox/tox.ini

@@ -57,6 +57,7 @@ requires=
 setenv =
   SPHINX_APIDOC_OPTIONS=members,undoc-members,inherited-members
   PIP_EXTRA_INDEX_URL=https://pypi.python.org/simple
+  PROXY_URL=http://localhost:5001
   VIRTUALENV_WHEEL=0.37.0
   VIRTUALENV_PIP=20.3.3
   VIRTUALENV_SETUPTOOLS=59.6.0
@@ -78,6 +79,9 @@ skipsdist = false
 skip_install = false
 usedevelop = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5002
 deps =
   {[base]deps}
   -e {toxinidir}/../../scripts/pylint_custom_plugin
@@ -90,6 +94,9 @@ skipsdist = false
 skip_install = false
 usedevelop = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5003
 deps =
   {[base]deps}
   mypy==0.782; python_version >= '3.5'
@@ -103,6 +110,9 @@ skip_install = true
 changedir = {toxinidir}
 deps =
   {[base]deps}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5004
 commands =
     - {envbindir}/python -m pip uninstall aiohttp --yes
     {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir} -w {envtmpdir}
@@ -119,6 +129,9 @@ commands =
 skipsdist = true
 skip_install = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5005
 deps =
   {[base]deps}
 commands =
@@ -142,6 +155,9 @@ usedevelop = true
 changedir = {toxinidir}
 deps =
   {[base]deps}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5006
 commands =
     pytest \
       {posargs} \
@@ -154,6 +170,9 @@ skipsdist = false
 skip_install = false
 changedir = {toxinidir}
 passenv = *
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5007
 deps =
   {[base]deps}
   sphinx==3.5.4
@@ -178,6 +197,9 @@ platform = linux: linux
            macos: darwin
            windows: win32
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5008
 deps =
   {[packaging]pkgs}
   requests
@@ -197,6 +219,9 @@ commands =
 skipsdist = true
 skip_install = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5009
 deps =
     {[packaging]pkgs}
 commands =
@@ -208,6 +233,9 @@ commands =
 skipsdist = true
 skip_install = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5010
 deps =
     {[packaging]pkgs}
 commands =
@@ -218,6 +246,9 @@ commands =
 [testenv:devtest]
 deps = {[base]deps}
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5011
 commands =
     {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir}
     {envbindir}/python {toxinidir}/../../../eng/tox/install_dev_build_dependency.py -t {toxinidir}
@@ -247,6 +278,7 @@ passenv = *
 setenv =
   {[testenv]setenv}
   DEPENDENCY_TYPE=Latest
+  PROXY_URL=http://localhost:5012
 commands =
     {[deptestcommands]commands}
 
@@ -263,6 +295,7 @@ passenv = *
 setenv =
   {[testenv]setenv}
   DEPENDENCY_TYPE=Minimum
+  PROXY_URL=http://localhost:5013
 commands =
     {[deptestcommands]commands}
 
@@ -272,6 +305,9 @@ skipsdist = true
 skip_install = true
 usedevelop = false
 changedir = {envtmpdir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5014
 deps =
   {[base]deps}
 commands =
@@ -286,6 +322,9 @@ skipsdist = false
 skip_install = false
 usedevelop = false
 changedir = {envtmpdir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5015
 deps =
   {[base]deps}
 commands =
@@ -298,6 +337,9 @@ skipsdist = false
 skip_install = false
 usedevelop = false
 changedir = {envtmpdir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5016
 deps =
   {[base]deps}
   subprocess32; python_version < '3.5'
@@ -311,6 +353,9 @@ skipsdist = false
 skip_install = false
 usedevelop = true
 changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5017
 deps =
   {[base]deps}
   jsondiff==1.2.0

tools/azure-sdk-tools/devtools_testutils/__init__.py

@@ -18,7 +18,7 @@
 # cSpell:disable
 from .envvariable_loader import EnvironmentVariableLoader
 PowerShellPreparer = EnvironmentVariableLoader  # Backward compat
-from .proxy_docker_startup import start_test_proxy, stop_test_proxy, test_proxy
+from .proxy_startup import start_test_proxy, stop_test_proxy, test_proxy
 from .proxy_testcase import recorded_by_proxy
 from .sanitizers import (
     add_body_key_sanitizer,

tools/azure-sdk-tools/devtools_testutils/config.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+import os
 
-
-PROXY_URL = "https://localhost:5001"
+PROXY_URL = os.getenv("PROXY_URL", "https://localhost:5001")
 TEST_SETTING_FILENAME = "testsettings_local.cfg"