[Draft] Integration tests perf

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/Program.cs

@@ -8,7 +8,6 @@
 // Opt-out telemetry
 Environment.SetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT", "1");
 
-CommandLine.MaxOutstandingCommands = Environment.ProcessorCount;
 DotnetCli.DoNotRetry = Debugger.IsAttached;
 
 ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args);
@@ -29,5 +28,4 @@ CompositeExtensionFactory<SlowestTestsConsumer> slowestTestCompositeServiceFacto
 builder.TestHost.AddTestSessionLifetimeHandle(slowestTestCompositeServiceFactory);
 using ITestApplication app = await builder.BuildAsync();
 int returnValue = await app.RunAsync();
-Console.WriteLine($"Process started: {CommandLine.TotalProcessesAttempt}");
 return returnValue;

test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs

@@ -8,7 +8,6 @@
 // Opt-out telemetry
 Environment.SetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT", "1");
 
-CommandLine.MaxOutstandingCommands = Environment.ProcessorCount;
 DotnetCli.DoNotRetry = Debugger.IsAttached;
 
 ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args);
@@ -30,5 +29,4 @@ CompositeExtensionFactory<SlowestTestsConsumer> slowestTestCompositeServiceFacto
 builder.TestHost.AddTestSessionLifetimeHandle(slowestTestCompositeServiceFactory);
 using ITestApplication app = await builder.BuildAsync();
 int returnValue = await app.RunAsync();
-Console.WriteLine($"Process started: {CommandLine.TotalProcessesAttempt}");
 return returnValue;

test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/ServerMode/ServerModeTestsBase.cs

@@ -54,7 +54,7 @@ protected async Task<TestingPlatformClient> StartAsServerAndConnectToTheClientAs
             EnvironmentVariables = environmentVariables,
         };
 
-        IProcessHandle processHandler = ProcessFactory.Start(processConfig, cleanDefaultEnvironmentVariableIfCustomAreProvided: false);
+        (IProcessHandle processHandler, _) = ProcessFactory.Start(processConfig, cleanDefaultEnvironmentVariableIfCustomAreProvided: false);
 
         TcpClient? tcpClient;
         using CancellationTokenSource cancellationTokenSource = new(TimeSpan.FromSeconds(60));

test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs

@@ -7,13 +7,6 @@
 
 public sealed class CommandLine : IDisposable
 {
-    private static int s_totalProcessesAttempt;
-    [SuppressMessage("Style", "IDE0032:Use auto property", Justification = "It's causing some runtime issue")]
-    private static int s_maxOutstandingCommand = Environment.ProcessorCount;
-    private static SemaphoreSlim s_maxOutstandingCommands_semaphore = new(s_maxOutstandingCommand, s_maxOutstandingCommand);
-
-    public static int TotalProcessesAttempt => s_totalProcessesAttempt;
-
     private readonly List<string> _errorOutputLines = [];
     private readonly List<string> _standardOutputLines = [];
     private IProcessHandle? _process;
@@ -26,18 +19,6 @@
 
     public string ErrorOutput => string.Join(Environment.NewLine, _errorOutputLines);
 
-    public static int MaxOutstandingCommands
-    {
-        get => s_maxOutstandingCommand;
-
-        set
-        {
-            s_maxOutstandingCommand = value;
-            s_maxOutstandingCommands_semaphore.Dispose();
-            s_maxOutstandingCommands_semaphore = new SemaphoreSlim(s_maxOutstandingCommand, s_maxOutstandingCommand);
-        }
-    }
-
     public async Task RunAsync(
         string commandLine,
         IDictionary<string, string?>? environmentVariables = null,
@@ -46,7 +27,7 @@
         int exitCode = await RunAsyncAndReturnExitCodeAsync(commandLine, environmentVariables, cancellationToken: cancellationToken);
         if (exitCode != 0)
         {
             throw new InvalidOperationException(
                 $"""
                     Non-zero exit code {exitCode} from command line: '{commandLine}'
                     STD: {StandardOutput}
@@ -75,30 +56,22 @@
         bool cleanDefaultEnvironmentVariableIfCustomAreProvided = false,
         CancellationToken cancellationToken = default)
     {
-        await s_maxOutstandingCommands_semaphore.WaitAsync(cancellationToken);
-        try
+        (string command, string arguments) = GetCommandAndArguments(commandLine);
+        _errorOutputLines.Clear();
+        _standardOutputLines.Clear();
+        ProcessConfiguration startInfo = new(command)
         {
-            Interlocked.Increment(ref s_totalProcessesAttempt);
-            (string command, string arguments) = GetCommandAndArguments(commandLine);
-            _errorOutputLines.Clear();
-            _standardOutputLines.Clear();
-            ProcessConfiguration startInfo = new(command)
-            {
-                Arguments = arguments,
-                EnvironmentVariables = environmentVariables,
-                OnErrorOutput = (_, o) => _errorOutputLines.Add(ClearBOM(o)),
-                OnStandardOutput = (_, o) => _standardOutputLines.Add(ClearBOM(o)),
-                WorkingDirectory = workingDirectory,
-            };
-            _process = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided);
-
-            using CancellationTokenRegistration registration = cancellationToken.Register(() => _process.Kill());
-            return await _process.WaitForExitAsync(cancellationToken);
-        }
-        finally
-        {
-            s_maxOutstandingCommands_semaphore.Release();
-        }
+            Arguments = arguments,
+            EnvironmentVariables = environmentVariables,
+            OnErrorOutput = (_, o) => _errorOutputLines.Add(ClearBOM(o)),
+            OnStandardOutput = (_, o) => _standardOutputLines.Add(ClearBOM(o)),
+            WorkingDirectory = workingDirectory,
+        };
+        (_process, Task outputAndErrorTask) = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided);
+
+        using CancellationTokenRegistration registration = cancellationToken.Register(() => _process.Kill());
+        await outputAndErrorTask;
+        return await _process.WaitForExitAsync(cancellationToken);
     }
 
     /// <summary>
@@ -113,5 +86,11 @@
         return firstChar == byteOrderMark ? outputLine[1..] : outputLine;
     }
 
-    public void Dispose() => _process?.Kill();
+    public void Dispose()
+    {
+        if (_process?.HasExited == false)
+        {
+            _process.Kill();
+        }
+    }
 }

test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs

@@ -29,22 +29,6 @@ public static class DotnetCli
 
     private static int s_binlogCounter;
 
-    [SuppressMessage("Style", "IDE0032:Use auto property", Justification = "It's causing some runtime bug")]
-    private static int s_maxOutstandingCommand = Environment.ProcessorCount;
-    private static SemaphoreSlim s_maxOutstandingCommands_semaphore = new(s_maxOutstandingCommand, s_maxOutstandingCommand);
-
-    public static int MaxOutstandingCommands
-    {
-        get => s_maxOutstandingCommand;
-
-        set
-        {
-            s_maxOutstandingCommand = value;
-            s_maxOutstandingCommands_semaphore.Dispose();
-            s_maxOutstandingCommands_semaphore = new SemaphoreSlim(s_maxOutstandingCommand, s_maxOutstandingCommand);
-        }
-    }
-
     public static bool DoNotRetry { get; set; }
 
     public static async Task<DotnetMuxerResult> RunAsync(
@@ -61,69 +45,61 @@ public static async Task<DotnetMuxerResult> RunAsync(
         [CallerMemberName] string callerMemberName = "",
         CancellationToken cancellationToken = default)
     {
-        await s_maxOutstandingCommands_semaphore.WaitAsync(cancellationToken);
-        try
+        environmentVariables ??= [];
+        foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables())
         {
-            environmentVariables ??= [];
-            foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables())
+            // Skip all unwanted environment variables.
+            string? key = entry.Key.ToString();
+            if (WellKnownEnvironmentVariables.ToSkipEnvironmentVariables.Contains(key, StringComparer.OrdinalIgnoreCase))
             {
-                // Skip all unwanted environment variables.
-                string? key = entry.Key.ToString();
-                if (WellKnownEnvironmentVariables.ToSkipEnvironmentVariables.Contains(key, StringComparer.OrdinalIgnoreCase))
-                {
-                    continue;
-                }
+                continue;
+            }
 
-                if (disableCodeCoverage)
+            if (disableCodeCoverage)
+            {
+                // Disable the code coverage during the build.
+                if (CodeCoverageEnvironmentVariables.Contains(key, StringComparer.OrdinalIgnoreCase))
                 {
-                    // Disable the code coverage during the build.
-                    if (CodeCoverageEnvironmentVariables.Contains(key, StringComparer.OrdinalIgnoreCase))
-                    {
-                        continue;
-                    }
+                    continue;
                 }
-
-                // We use TryAdd to let tests "overwrite" existing environment variables.
-                // Consider that the given dictionary has "TESTINGPLATFORM_UI_LANGUAGE" as a key.
-                // And also Environment.GetEnvironmentVariables() is returning TESTINGPLATFORM_UI_LANGUAGE.
-                // In that case, we do a "TryAdd" which effectively means the value from the original dictionary wins.
-                environmentVariables.TryAdd(key!, entry.Value!.ToString()!);
             }
 
-            if (disableTelemetry)
-            {
-                environmentVariables.Add("DOTNET_CLI_TELEMETRY_OPTOUT", "1");
-            }
+            // We use TryAdd to let tests "overwrite" existing environment variables.
+            // Consider that the given dictionary has "TESTINGPLATFORM_UI_LANGUAGE" as a key.
+            // And also Environment.GetEnvironmentVariables() is returning TESTINGPLATFORM_UI_LANGUAGE.
+            // In that case, we do a "TryAdd" which effectively means the value from the original dictionary wins.
+            environmentVariables.TryAdd(key!, entry.Value!.ToString()!);
+        }
+
+        if (disableTelemetry)
+        {
+            environmentVariables.Add("DOTNET_CLI_TELEMETRY_OPTOUT", "1");
+        }
 
-            environmentVariables["NUGET_PACKAGES"] = nugetGlobalPackagesFolder;
+        environmentVariables["NUGET_PACKAGES"] = nugetGlobalPackagesFolder;
 
-            string extraArgs = warnAsError ? " -p:MSBuildTreatWarningsAsErrors=true" : string.Empty;
-            extraArgs += suppressPreviewDotNetMessage ? " -p:SuppressNETCoreSdkPreviewMessage=true" : string.Empty;
-            if (args.IndexOf("-- ", StringComparison.Ordinal) is int platformArgsIndex && platformArgsIndex > 0)
-            {
-                args = args.Insert(platformArgsIndex, extraArgs + " ");
-            }
-            else
-            {
-                args += extraArgs;
-            }
+        string extraArgs = warnAsError ? " -p:MSBuildTreatWarningsAsErrors=true" : string.Empty;
+        extraArgs += suppressPreviewDotNetMessage ? " -p:SuppressNETCoreSdkPreviewMessage=true" : string.Empty;
+        if (args.IndexOf("-- ", StringComparison.Ordinal) is int platformArgsIndex && platformArgsIndex > 0)
+        {
+            args = args.Insert(platformArgsIndex, extraArgs + " ");
+        }
+        else
+        {
+            args += extraArgs;
+        }
 
-            if (DoNotRetry)
-            {
-                return await CallTheMuxerAsync(args, environmentVariables, workingDirectory, failIfReturnValueIsNotZero, callerMemberName, cancellationToken);
-            }
-            else
-            {
-                IEnumerable<TimeSpan> delay = Backoff.ExponentialBackoff(TimeSpan.FromSeconds(3), retryCount, factor: 1.5);
-                return await Policy
-                    .Handle<Exception>()
-                    .WaitAndRetryAsync(delay)
-                    .ExecuteAsync(async ct => await CallTheMuxerAsync(args, environmentVariables, workingDirectory, failIfReturnValueIsNotZero, callerMemberName, ct), cancellationToken);
-            }
+        if (DoNotRetry)
+        {
+            return await CallTheMuxerAsync(args, environmentVariables, workingDirectory, failIfReturnValueIsNotZero, callerMemberName, cancellationToken);
         }
-        finally
+        else
         {
-            s_maxOutstandingCommands_semaphore.Release();
+            IEnumerable<TimeSpan> delay = Backoff.ExponentialBackoff(TimeSpan.FromSeconds(3), retryCount, factor: 1.5);
+            return await Policy
+                .Handle<Exception>()
+                .WaitAndRetryAsync(delay)
+                .ExecuteAsync(async ct => await CallTheMuxerAsync(args, environmentVariables, workingDirectory, failIfReturnValueIsNotZero, callerMemberName, ct), cancellationToken);
         }
     }
 

test/Utilities/Microsoft.Testing.TestInfrastructure/IProcessHandle.cs

@@ -15,6 +15,8 @@ public interface IProcessHandle
 
     TextReader StandardOutput { get; }
 
+    bool HasExited { get; }
+
     void Dispose();
 
     void Kill();

test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs

@@ -5,7 +5,7 @@ namespace Microsoft.Testing.TestInfrastructure;
 
 public static class ProcessFactory
 {
-    public static IProcessHandle Start(ProcessConfiguration config, bool cleanDefaultEnvironmentVariableIfCustomAreProvided = false)
+    public static (IProcessHandle Handle, Task OutputAndErrorTask) Start(ProcessConfiguration config, bool cleanDefaultEnvironmentVariableIfCustomAreProvided = false)
     {
         string fullPath = config.FileName; // Path.GetFullPath(startInfo.FileName);
         string workingDirectory = config.WorkingDirectory
@@ -61,32 +61,34 @@ public static IProcessHandle Start(ProcessConfiguration config, bool cleanDefaul
             process.Exited += (s, e) => config.OnExit.Invoke(processHandle, process.ExitCode);
         }
 
-        if (config.OnStandardOutput != null)
+        if (!process.Start())
         {
-            process.OutputDataReceived += (s, e) =>
+            throw new InvalidOperationException("Process failed to start");
+        }
+
+        Task outputTask = Task.Factory.StartNew(
+            () =>
             {
-                if (!string.IsNullOrWhiteSpace(e.Data))
+                while (process.StandardOutput.ReadLine() is string line)
                 {
-                    config.OnStandardOutput(processHandle, e.Data);
+                    if (!string.IsNullOrWhiteSpace(line))
+                    {
+                        config.OnStandardOutput?.Invoke(processHandle, line);
+                    }
                 }
-            };
-        }
+            }, TaskCreationOptions.LongRunning);
 
-        if (config.OnErrorOutput != null)
-        {
-            process.ErrorDataReceived += (s, e) =>
+        Task errorTask = Task.Factory.StartNew(
+            () =>
             {
-                if (!string.IsNullOrWhiteSpace(e.Data))
+                while (process.StandardError.ReadLine() is string line)
                 {
-                    config.OnErrorOutput(processHandle, e.Data);
+                    if (!string.IsNullOrWhiteSpace(line))
+                    {
+                        config.OnErrorOutput?.Invoke(processHandle, line);
+                    }
                 }
-            };
-        }
-
-        if (!process.Start())
-        {
-            throw new InvalidOperationException("Process failed to start");
-        }
+            }, TaskCreationOptions.LongRunning);
 
         try
         {
@@ -100,16 +102,6 @@ public static IProcessHandle Start(ProcessConfiguration config, bool cleanDefaul
 
         processHandleInfo.Id = process.Id;
 
-        if (config.OnStandardOutput != null)
-        {
-            process.BeginOutputReadLine();
-        }
-
-        if (config.OnErrorOutput != null)
-        {
-            process.BeginErrorReadLine();
-        }
-
-        return processHandle;
+        return (processHandle, Task.WhenAll(outputTask, errorTask));
     }
 }

test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessHandle.cs

@@ -24,6 +24,8 @@ internal ProcessHandle(Process process, ProcessHandleInfo processHandleInfo)
 
     public TextReader StandardOutput => _process.StandardOutput;
 
+    public bool HasExited => _process.HasExited;
+
     public int ExitCode => _process.ExitCode;
 
     public async Task<int> WaitForExitAsync(CancellationToken cancellationToken = default)