[DRAFT] Xunit2 -> Xunit3

CustomAnalysisRules.Test.ruleset

@@ -129,4 +129,12 @@
     <Rule Id="CA2200" Action="None" />
     <Rule Id="CA2234" Action="None" />
   </Rules>
+  <Rules AnalyzerId="xUnit.Analyzers" RuleNamespace="xUnit.Analyzers">
+    <Rule Id="xUnit1031" Action="None" />
+    <Rule Id="xUnit1051" Action="None" />
+  </Rules>
+  <Rules AnalyzerId="StyleCop.Analyzers" RuleNamespace="StyleCop.Analyzers">
+    <Rule Id="SA1208" Action="None" />
+    <Rule Id="SA1209" Action="None" />
+  </Rules>
 </RuleSet>

Directory.Build.props

@@ -29,14 +29,16 @@
   <Choose>
     <When Condition="$(MSBuildProjectName.Contains('UnitTest'))">
       <ItemGroup>
-        <Content Include="$(MSBuildThisFileDirectory)/xunit.runner.json">
+        <None Include="$(MSBuildThisFileDirectory)/xunit.runner.json">
           <Link>xunit.runner.json</Link>
           <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-        </Content>
+          <CopyToPublishDirectory>Never</CopyToPublishDirectory>
+        </None>
       </ItemGroup>
     </When>
   </Choose>
 
+  <!-- Test project configuration for xUnit 3 and Microsoft Testing Platform -->
   <Choose>
     <When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' != 'false'">
       <PropertyGroup>
@@ -45,13 +47,15 @@
         <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
         <TestingPlatformCaptureOutput>false</TestingPlatformCaptureOutput>
         <OutputType>Exe</OutputType>
+        <UseMicrosoftTestingPlatformRunner>true</UseMicrosoftTestingPlatformRunner>
+        <NoWarn>$(NoWarn);xUnit1031;xUnit1051;SA1208;SA1209;SA1518</NoWarn>
       </PropertyGroup>
       <ItemGroup>
-        <PackageReference Include="YTest.MTP.XUnit2" />
         <PackageReference Include="Microsoft.Testing.Extensions.Retry" />
         <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
         <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
-        <PackageReference Remove="xunit.runner.visualstudio" />
+        <PackageReference Include="Microsoft.Testing.Extensions.Telemetry" />
+        <PackageReference Include="Microsoft.Testing.Platform.MSBuild" />
       </ItemGroup>
     </When>
     <When Condition="$(MSBuildProjectName.Contains('Test')) AND '$(IsTestProject)' == 'false'">

Directory.Packages.props

@@ -39,8 +39,10 @@
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.4.0" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version="1.2.0" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
-    <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.1.0" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.1" />
     <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.Telemetry" Version="2.0.2" />
+    <PackageVersion Include="Microsoft.Testing.Platform.MSBuild" Version="2.0.2" />
     <PackageVersion Include="DotNet.ReproducibleBuilds" Version="1.2.25" />
     <PackageVersion Include="Ensure.That" Version="10.1.0" />
     <PackageVersion Include="FluentValidation" Version="11.11.0" />
@@ -132,13 +134,13 @@
     <PackageVersion Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
     <PackageVersion Include="System.Private.ServiceModel" Version="4.10.3" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
-    <PackageVersion Include="YTest.MTP.XUnit2" Version="1.0.3" />
+    <PackageVersion Include="xunit.v3" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.assert" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.common" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.extensibility.core" Version="3.2.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
     <PackageVersion Include="Microsoft.Testing.Extensions.Retry" Version="2.0.2" />
-    <PackageVersion Include="xunit.extensibility.core" Version="2.9.3" />
-    <PackageVersion Include="Xunit.SkippableFact" Version="1.5.23" />
-    <PackageVersion Include="xunit" Version="2.9.3" />
-    <PackageVersion Include="xunit.assert" Version="2.9.3" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.8" />
     <PackageVersion Include="Moq" Version="4.20.69" />
     <PackageVersion Include="System.Text.RegularExpressions" Version="4.3.1" />

THIRDPARTYNOTICES.md

@@ -969,58 +969,3 @@ This file is based on or incorporates material from the projects listed below (T
   > WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   > See the License for the specific language governing permissions and
   > limitations under the License.
-
-## YTest.MTP.XUnit2 1.0.3
-* Component Source:   https://github.com/Youssef1313/YTest.MTP.XUnit2
-* Component Copyright and License:
-  > The MIT License (MIT)
-  > 
-  > Copyright (c) Youssef Victor Gerges Fahmy
-  > 
-  > All rights reserved.
-  > 
-  > Permission is hereby granted, free of charge, to any person obtaining a copy
-  > of this software and associated documentation files (the "Software"), to deal
-  > in the Software without restriction, including without limitation the rights
-  > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-  > copies of the Software, and to permit persons to whom the Software is
-  > furnished to do so, subject to the following conditions:
-  > 
-  > The above copyright notice and this permission notice shall be included in all
-  > copies or substantial portions of the Software.
-  > 
-  > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-  > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-  > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-  > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-  > SOFTWARE.
-
-## Xunit.SkippableFact 1.4.13
-* Component Source:   https://github.com/AArnott/Xunit.SkippableFact
-* Component Copyright and License:  
- > This software is licensed under the Microsoft Public License
- > http://opensource.org/licenses/ms-pl
- > 
- > This license governs use of the accompanying software. If you use the software, you
- > accept this license. If you do not accept the license, do not use the software.
- > 
- > 1. Definitions
- > The terms "reproduce," "reproduction," "derivative works," and "distribution" have the
- > same meaning here as under U.S. copyright law.
- > A "contribution" is the original software, or any additions or changes to the software.
- > A "contributor" is any person that distributes its contribution under this license.
- > "Licensed patents" are a contributor's patent claims that read directly on its contribution.
- > 
- > 2. Grant of Rights
- > (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create.
- > (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software.
- > 
- > 3. Conditions and Limitations
- > (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks.
- > (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically.
- > (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software.
- > (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license.
- > (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement.
-

build/jobs/build.yml

@@ -35,19 +35,41 @@ steps:
     inputs:
       command: test
       projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3'
+      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx'
+      publishTestResults: false
+    env:
+      platformOptions__resultDirectory: '$(Agent.TempDirectory)/unit-test-results'
+  - task: PublishTestResults@2
+    displayName: 'Publish unit test results'
+    inputs:
+      testResultsFormat: 'VSTest'
+      testResultsFiles: '$(Agent.TempDirectory)/unit-test-results/**/*.trx'
+      mergeTestResults: true
       testRunTitle: 'Unit Tests'
+      # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed
+      failTaskOnFailedTests: false
+    condition: succeededOrFailed()
 
 - ${{ if eq(parameters.codeCoverage, 'true') }}:
   - task: DotNetCoreCLI@2
     displayName: 'dotnet test with coverage'
     inputs:
       command: test
       projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"'
-      testRunTitle: 'Unit Tests'
+      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}} -- --retry-failed-tests 3 --report-trx --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml"'
+      publishTestResults: false
     env:
       platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'
+  - task: PublishTestResults@2
+    displayName: 'Publish unit test with coverage results'
+    inputs:
+      testResultsFormat: 'VSTest'
+      testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx'
+      mergeTestResults: true
+      testRunTitle: 'Unit Tests'
+      # TODO: Re-enable when https://github.com/microsoft/testfx/issues/7167 is fixed
+      failTaskOnFailedTests: false
+    condition: succeededOrFailed()
   - task: reportgenerator@5
     displayName: 'aggregate unit test coverage'
     condition: succeededOrFailed()

build/jobs/e2e-tests.yml

@@ -37,10 +37,15 @@ steps:
           throw "Could not find $testName.dll under $testRoot"
         }
 
-        $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}"
-        $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx')
+        # xUnit v3 MTP filter-query: match DataStore trait for this app service type and any extra trait filters
+        $categoryFilter = '${{ parameters.categoryFilter }}'
+        $parts = $categoryFilter -split '&'
+        $allParts = @("DataStore=${{ parameters.appServiceType }}") + $parts
+        $traitFilter = "[" + (($allParts | ForEach-Object { "($_)" }) -join '&') + "]"
+        $filterQuery = "/$traitFilter"
+        $args = @('--filter-query', $filterQuery, '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx')
 
-        Write-Host "Running dotnet $($dll.FullName)"
+        Write-Host "Running dotnet $($dll.FullName) with filter: $filterQuery"
         & dotnet $dll.FullName @args
 
         if ($LASTEXITCODE -ne 0) {

build/jobs/run-cosmos-tests.yml

@@ -68,13 +68,36 @@ jobs:
       projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
       arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)'
 
+  - task: PowerShell@2
+    displayName: 'Verify xUnit config'
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        $ErrorActionPreference = 'Stop'
+        $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj"
+        if (-not $projects) {
+          throw "No integration test projects found under $(Pipeline.Workspace)/source/test."
+        }
+
+        foreach ($project in $projects) {
+          $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)")
+          $configPath = Join-Path $outputDir "xunit.runner.json"
+          if (-not (Test-Path $configPath)) {
+            throw "xunit.runner.json not found for $($project.FullName) at $configPath"
+          }
+
+          Write-Host "xUnit config for $($project.BaseName): $configPath"
+          Get-Content -Path $configPath | ForEach-Object { Write-Host $_ }
+        }
+
   - task: DotNetCoreCLI@2
     displayName: 'Run Cosmos Integration Tests with coverage'
     inputs:
       command: test
       projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
-      testRunTitle: '${{ parameters.version }} Cosmos Integration Tests'
+      arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*SqlServer*" --filter-not-trait "DataStore=SqlServer" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
+      publishTestResults: false
     env:
       'CosmosDb__Host': $(CosmosDb--Host)
       'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)'

build/jobs/run-export-tests.yml

@@ -100,9 +100,9 @@ jobs:
           throw "Could not find $testName.dll under $testRoot"
         }
 
-        $args = @('--filter', 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx')
+        $args = @('--filter-query', '/[(DataStore=CosmosDb)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx')
 
-        Write-Host "Running dotnet $($dll.FullName)"
+        Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=CosmosDb)&(Category=ExportLongRunning)]"
         & dotnet $dll.FullName @args
 
         if ($LASTEXITCODE -ne 0) {
@@ -240,9 +240,9 @@ jobs:
           throw "Could not find $testName.dll under $testRoot"
         }
 
-        $args = @('--filter', 'FullyQualifiedName~SqlServer&Category=ExportLongRunning', '--retry-failed-tests', '3', '--report-trx')
+        $args = @('--filter-query', '/[(DataStore=SqlServer)&(Category=ExportLongRunning)]', '--parallel', 'none', '--retry-failed-tests', '3', '--report-trx')
 
-        Write-Host "Running dotnet $($dll.FullName)"
+        Write-Host "Running dotnet $($dll.FullName) with filter: /[(DataStore=SqlServer)&(Category=ExportLongRunning)]"
         & dotnet $dll.FullName @args
 
         if ($LASTEXITCODE -ne 0) {

build/jobs/run-sql-tests.yml

@@ -60,13 +60,36 @@ jobs:
       projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
       arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)'
 
+  - task: PowerShell@2
+    displayName: 'Verify xUnit config'
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        $ErrorActionPreference = 'Stop'
+        $projects = Get-ChildItem -Path "$(Pipeline.Workspace)/source/test" -Recurse -Filter "*${{ parameters.version }}.Tests.Integration.csproj"
+        if (-not $projects) {
+          throw "No integration test projects found under $(Pipeline.Workspace)/source/test."
+        }
+
+        foreach ($project in $projects) {
+          $outputDir = [System.IO.Path]::Combine($project.Directory.FullName, "bin", "$(buildConfiguration)", "$(defaultBuildFramework)")
+          $configPath = Join-Path $outputDir "xunit.runner.json"
+          if (-not (Test-Path $configPath)) {
+            throw "xunit.runner.json not found for $($project.FullName) at $configPath"
+          }
+
+          Write-Host "xUnit config for $($project.BaseName): $configPath"
+          Get-Content -Path $configPath | ForEach-Object { Write-Host $_ }
+        }
+
   - task: DotNetCoreCLI@2
     displayName: 'Run SQL Integration Tests with coverage'
     inputs:
       command: test
       projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "FullyQualifiedName!~CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
-      testRunTitle: '${{ parameters.version }} SQL Integration Tests'
+      arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter-not-class "*CosmosDb*" --filter-not-trait "DataStore=CosmosDb" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
+      publishTestResults: false
     env:
       'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);'
       platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'

src/Microsoft.Health.Extensions.Xunit/AssemblyFixtureAttribute.cs

@@ -4,24 +4,12 @@
 // -------------------------------------------------------------------------------------------------
 
 using System;
-using EnsureThat;
 
 namespace Microsoft.Health.Extensions.Xunit
 {
-    /// <summary>
-    /// Placed on an assembly to indicate that the given type should be instantiated
-    /// before any tests are executed and disposed (if it implements IDisposable)
-    /// at the end of the test run.
-    /// </summary>
-    [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
+    [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true, Inherited = false)]
+    [Obsolete("Use Xunit.AssemblyFixtureAttribute instead.")]
     public sealed class AssemblyFixtureAttribute : Attribute
     {
-        public AssemblyFixtureAttribute(Type fixtureType)
-        {
-            EnsureArg.IsNotNull(fixtureType, nameof(fixtureType));
-            FixtureType = fixtureType;
-        }
-
-        public Type FixtureType { get; }
     }
 }