Initial push of CLI command line tests

eng/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml

@@ -36,43 +36,9 @@ stages:
           vmImage: $(Image)
 
         steps:
-          - pwsh: |
-              Install-Module -Name Pester -Force
-            displayName: Install Pester
-
-          # default test steps
-          - ${{ if eq(length(parameters.CustomTestSteps), 0) }}:
-            - pwsh: |
-                $tags = "${{ parameters.TargetTags }}" -Split "," | ForEach-Object { return $_.Trim() }
-
-                $config = New-PesterConfiguration
-                $config.CodeCoverage.Enabled = $true
-                $config.TestResult.Enabled = $true
-
-                if ($tags) {
-                  $config.Filter.Tag = $tags
-                }
-
-                Invoke-Pester -Configuration $config
-              displayName: Run Tests
-              env: ${{ parameters.EnvVars }}
-              workingDirectory: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}
-
-          - ${{ if not(eq(length(parameters.CustomTestSteps), 0)) }}:
-            - ${{ parameters.CustomTestSteps }}
-
-          - task: PublishTestResults@2
-            displayName: 'Publish Test Results'
-            condition: succeededOrFailed()
-            inputs:
-              testResultsFormat: 'NUnit'
-              testResultsFiles: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/testResults.xml
-              testRunTitle: 'Tests_$(Image)'
-
-          - task: PublishCodeCoverageResults@1
-            displayName: 'Publish Code Coverage to Azure DevOps'
-            condition: succeededOrFailed()
-            inputs:
-              codeCoverageTool: 'JaCoCo'
-              summaryFileLocation: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/coverage.xml'
-              pathToSources: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}'
+          - template: /eng/pipelines/templates/steps/run-pester-tests.yml
+            parameters:
+              TargetDirectory: ${{ parameters.TargetDirectory }}
+              CustomTestSteps: ${{ parameters.CustomTestSteps }}
+              EnvVars: ${{ parameters.EnvVars }}
+              TargetTags: ${{ parameters.TargetTags }}

eng/pipelines/templates/steps/run-pester-tests.yml

@@ -0,0 +1,54 @@
+parameters:
+  - name: TargetDirectory
+    type: string
+  - name: CustomTestSteps
+    type: object
+    default: []
+  - name: EnvVars
+    type: object
+    default: {}
+  - name: TargetTags
+    type: string
+    default: ''
+
+steps:
+  - pwsh: |
+     Install-Module -Name Pester -Force
+    displayName: Install Pester
+
+   # default test steps
+  - ${{ if eq(length(parameters.CustomTestSteps), 0) }}:
+    - pwsh: |
+        $tags = "${{ parameters.TargetTags }}" -Split "," | ForEach-Object { return $_.Trim() }
+
+        $config = New-PesterConfiguration
+        $config.CodeCoverage.Enabled = $true
+        $config.TestResult.Enabled = $true
+
+        if ($tags) {
+          $config.Filter.Tag = $tags
+        }
+
+        Invoke-Pester -Configuration $config
+      displayName: Run Tests
+      env: ${{ parameters.EnvVars }}
+      workingDirectory: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}
+
+  - ${{ if not(eq(length(parameters.CustomTestSteps), 0)) }}:
+    - ${{ parameters.CustomTestSteps }}
+
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFormat: 'NUnit'
+      testResultsFiles: $(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/testResults.xml
+      testRunTitle: '$(System.StageName)_$(Agent.JobName)_Tests'
+
+  - task: PublishCodeCoverageResults@1
+    displayName: 'Publish Code Coverage to Azure DevOps'
+    condition: succeededOrFailed()
+    inputs:
+      codeCoverageTool: 'JaCoCo'
+      summaryFileLocation: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}/coverage.xml'
+      pathToSources: '$(Build.SourcesDirectory)/${{ parameters.TargetDirectory }}'

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationPushTests.cs

@@ -13,6 +13,11 @@
 
 namespace Azure.Sdk.Tools.TestProxy.Tests.IntegrationTests
 {
+
+    // With the use of tags, the push scenarios are going to be somewhat redundant since
+    // tags are vastly simplified, in terms of processing compared to branches. The
+    // Scenarios below are similar but use 3 different starting tags with different
+    // files and different file versions.
     public class GitStoreIntegrationPushTests
     {
         public GitStoreIntegrationPushTests()
@@ -24,13 +29,11 @@ public GitStoreIntegrationPushTests()
         private GitStore _defaultStore = new GitStore();
 
         /// <summary>
-        /// New push scenario
-        /// 1. Auto Branch Doesn't Exist Yet
-        /// 2. New branch is created off of main
-        /// 3. Add/Delete/Update files
-        /// 4. Push to new branch
-        /// 5. Verify local files are what is expected
-        /// 6. Verify assets.json was updated with the new commit Tag
+        /// 1. Restore from the initial tag in pull/scenarios
+        /// 2. Add/Delete/Update files
+        /// 3. Push to new branch
+        /// 4. Verify local files are what is expected
+        /// 5. Verify assets.json was updated with the new commit Tag
         /// </summary>
         /// <param name="inputJson"></param>
         /// <returns></returns>
@@ -44,21 +47,21 @@ public GitStoreIntegrationPushTests()
               ""Tag"": ""language/tables_fc54d0""
         }")]
         [Trait("Category", "Integration")]
-        public async Task ScenarioNewPush(string inputJson)
+        public async Task Scenario1(string inputJson)
         {
             var folderStructure = new string[]
             {
                 GitStoretests.AssetsJson
             };
             Assets assets = JsonSerializer.Deserialize<Assets>(inputJson);
             Assets updatedAssets = null;
-            string originalAssetsRepoBranch = assets.TagPrefix;
-            string originalSHA = assets.Tag;
+            string originalTagPrefix = assets.TagPrefix;
+            string originalTag = assets.Tag;
             var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest:true);
             try
             {
                 // Ensure that the TagPrefix was updated
-                Assert.NotEqual(originalAssetsRepoBranch, assets.TagPrefix);
+                Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
 
                 var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
 
@@ -94,7 +97,7 @@ public async Task ScenarioNewPush(string inputJson)
 
                 // Ensure that the config was updated with the new Tag as part of the push
                 updatedAssets = TestHelpers.LoadAssetsFromFile(jsonFileLocation);
-                Assert.NotEqual(originalSHA, updatedAssets.Tag);
+                Assert.NotEqual(originalTag, updatedAssets.Tag);
 
                 // Ensure that the targeted tag is present on the repo
                 TestHelpers.CheckExistenceOfTag(updatedAssets, localFilePath);
@@ -107,10 +110,9 @@ public async Task ScenarioNewPush(string inputJson)
         }
 
         /// <summary>
-        /// Clean Push Scenario
-        /// 1. Branch already exists and we're on the latest Tag
+        /// 1. Restore from the second tag in pull/scenarios
         /// 2. Add/Delete/Update files
-        /// 3. Push commit to branch
+        /// 3. Push to new branch
         /// 4. Verify local files are what is expected
         /// 5. Verify assets.json was updated with the new commit Tag
         /// </summary>
@@ -126,21 +128,21 @@ public async Task ScenarioNewPush(string inputJson)
               ""Tag"": ""language/tables_bb2223""
         }")]
         [Trait("Category", "Integration")]
-        public async Task ScenarioCleanPush(string inputJson)
+        public async Task Scenario2(string inputJson)
         {
             var folderStructure = new string[]
             {
                 GitStoretests.AssetsJson
             };
             Assets assets = JsonSerializer.Deserialize<Assets>(inputJson);
             Assets updatedAssets = null;
-            string originalAssetsRepoBranch = assets.TagPrefix;
-            string originalSHA = assets.Tag;
+            string originalTagPrefix = assets.TagPrefix;
+            string originalTag = assets.Tag;
             var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
             try
             {
                 // Ensure that the TagPrefix was updated
-                Assert.NotEqual(originalAssetsRepoBranch, assets.TagPrefix);
+                Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
 
                 var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
 
@@ -176,7 +178,7 @@ public async Task ScenarioCleanPush(string inputJson)
 
                 // Ensure that the config was updated with the new Tag as part of the push
                 updatedAssets = TestHelpers.LoadAssetsFromFile(jsonFileLocation);
-                Assert.NotEqual(originalSHA, updatedAssets.Tag);
+                Assert.NotEqual(originalTag, updatedAssets.Tag);
 
                 // Ensure that the targeted tag is present on the repo
                 TestHelpers.CheckExistenceOfTag(updatedAssets, localFilePath);
@@ -189,10 +191,9 @@ public async Task ScenarioCleanPush(string inputJson)
         }
 
         /// <summary>
-        /// Conflict Push Scenario
-        /// 1. Branch already exists and we're not on the latest Tag
+        /// 1. Restore from the third tag in pull/scenarios
         /// 2. Add/Delete/Update files
-        /// 3. Push commit to branch, detects a conflict
+        /// 3. Push to new branch
         /// 4. Verify local files are what is expected
         /// 5. Verify assets.json was updated with the new commit Tag
         /// </summary>
@@ -208,21 +209,21 @@ public async Task ScenarioCleanPush(string inputJson)
               ""Tag"": ""language/tables_9e81fb""
         }")]
         [Trait("Category", "Integration")]
-        public async Task ScenarioConflictPush(string inputJson)
+        public async Task Scenario3(string inputJson)
         {
             var folderStructure = new string[]
             {
                 GitStoretests.AssetsJson
             };
             Assets assets = JsonSerializer.Deserialize<Assets>(inputJson);
             Assets updatedAssets = null;
-            string originalAssetsRepoBranch = assets.TagPrefix;
-            string originalSHA = assets.Tag;
+            string originalTagPrefix = assets.TagPrefix;
+            string originalTag = assets.Tag;
             var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
             try
             {
                 // Ensure that the TagPrefix was updated
-                Assert.NotEqual(originalAssetsRepoBranch, assets.TagPrefix);
+                Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
 
                 var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
 
@@ -269,7 +270,7 @@ public async Task ScenarioConflictPush(string inputJson)
 
                 // Ensure that the config was updated with the new Tag as part of the push
                 updatedAssets = TestHelpers.LoadAssetsFromFile(jsonFileLocation);
-                Assert.NotEqual(originalSHA, updatedAssets.Tag);
+                Assert.NotEqual(originalTag, updatedAssets.Tag);
 
                 // Ensure that the targeted tag is present on the repo
                 TestHelpers.CheckExistenceOfTag(updatedAssets, localFilePath);

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs

@@ -56,6 +56,7 @@ private static string resolveRepoLocation(string storageLocation = null)
         /// <param name="args">CommandLineParser arguments. In server mode use double dash '--' and everything after that becomes additional arguments to Host.CreateDefaultBuilder. Ex. -- arg1 value1 arg2 value2 </param>
         public static async Task Main(string[] args = null)
         {
+            VerifyVerb(args);
             // JRS - Temporarily disable this check because of https://github.com/Azure/azure-sdk-tools/issues/4116
             // This throws and will exit
             // new GitProcessHandler().VerifyGitMinVersion();
@@ -66,8 +67,55 @@ public static async Task Main(string[] args = null)
                 settings.HelpWriter = System.Console.Out;
                 settings.EnableDashDash = true;
             });
-
-            await parser.ParseArguments<StartOptions, PushOptions, ResetOptions, RestoreOptions>(args).WithParsedAsync(Run);
+
+            await parser.ParseArguments<StartOptions, PushOptions, ResetOptions, RestoreOptions>(args)
+                .WithNotParsed(ExitWithError)
+                .WithParsedAsync(Run);
+        }
+
+        static void ExitWithError(IEnumerable<Error> errors)
+        {
+            Environment.Exit(1);
+        }
+
+        /// <summary>
+        /// This is only necessary because if there's a default verb defined, ours is start,
+        /// CommandLineParser doesn't verify the verb. If the issue is fixed this function
+        /// can be removed.
+        /// https://github.com/commandlineparser/commandline/issues/849
+        /// </summary>
+        /// <param name="args"></param>
+        static void VerifyVerb(string[] args)
+        {
+            // no arguments means the server is starting with all the default options
+            if (args.Length == 0)
+            {
+                return;
+            }
+
+            // if the first argument starts with a dash then they're options and the
+            // default verb is being used.
+            if (args[0].StartsWith("-"))
+            {
+                return;
+            }
+
+            // last but not least, the first argument is a verb, verify it's our verb
+            string[] array = { "start", "reset", "restore", "push" };
+            if (!array.Contains(args[0]))
+            {
+                // The odd looking formatting is to make this look like the same error
+                // CommandLineParser would output if the verb wasn't recognized.
+                string error = @$"ERROR(S):
+  Verb '{args[0]}' is not recognized.
+
+  --help       Display this help screen.
+
+  --version    Display version information.
+";
+                System.Console.WriteLine(error);
+                Environment.Exit(1);
+            }
         }
 
         private static async Task Run(object commandObj)