Vulkan: Map RGB10A2 formats to A2B10G10R10

TEX_FORMAT_RGB10A2_* stores red in bits 0-9, like DXGI_FORMAT_R10G10B10A2 and
GL_RGB10_A2, which is VK_FORMAT_A2B10G10R10_*_PACK32. The A2R10G10B10 formats
used before swap red and blue, do not match the SPIR-V Rgb10A2/Rgb10a2ui image
formats and have no guaranteed storage image support, so RGB10A2 UAVs were
unavailable on devices such as the RTX 5060.

Add ClearRenderTargetTest.RGB10A2BitLayout, which checks the raw bits of a red
clear on every backend.

Author: hzqst

Graphics/GraphicsEngineVulkan/src/VulkanTypeConversions.cpp

@@ -74,9 +74,11 @@ class TexFormatToVkFormatMapper
         m_FmtToVkFmtMap[TEX_FORMAT_R32_FLOAT_X8X24_TYPELESS] = VK_FORMAT_D32_SFLOAT_S8_UINT;
         m_FmtToVkFmtMap[TEX_FORMAT_X32_TYPELESS_G8X24_UINT]  = VK_FORMAT_UNDEFINED;
 
-        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_TYPELESS] = VK_FORMAT_A2R10G10B10_UNORM_PACK32;
-        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_UNORM]    = VK_FORMAT_A2R10G10B10_UNORM_PACK32;
-        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_UINT]     = VK_FORMAT_A2R10G10B10_UINT_PACK32;
+        // RGB10A2 stores R in the low bits (DXGI_FORMAT_R10G10B10A2, GL_RGB10_A2), which is A2B10G10R10 in Vulkan.
+        // It is also the format of the SPIR-V Rgb10A2/Rgb10a2ui image formats and the one with guaranteed storage support.
+        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_TYPELESS] = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
+        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_UNORM]    = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
+        m_FmtToVkFmtMap[TEX_FORMAT_RGB10A2_UINT]     = VK_FORMAT_A2B10G10R10_UINT_PACK32;
         m_FmtToVkFmtMap[TEX_FORMAT_R11G11B10_FLOAT]  = VK_FORMAT_B10G11R11_UFLOAT_PACK32;
 
         m_FmtToVkFmtMap[TEX_FORMAT_RGBA8_TYPELESS]   = VK_FORMAT_R8G8B8A8_UNORM;
@@ -263,18 +265,18 @@ class VkFormatToTexFormatMapper
         m_VkFmtToTexFmtMap[VK_FORMAT_A8B8G8R8_SINT_PACK32]      = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A8B8G8R8_SRGB_PACK32]      = TEX_FORMAT_UNKNOWN;
 
-        m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_UNORM_PACK32]  = TEX_FORMAT_RGB10A2_UNORM;
+        m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_UNORM_PACK32]  = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_SNORM_PACK32]  = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_USCALED_PACK32]    = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_SSCALED_PACK32]    = TEX_FORMAT_UNKNOWN;
-        m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_UINT_PACK32]       = TEX_FORMAT_RGB10A2_UINT;
+        m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_UINT_PACK32]       = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2R10G10B10_SINT_PACK32]       = TEX_FORMAT_UNKNOWN;
 
-        m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_UNORM_PACK32]      = TEX_FORMAT_UNKNOWN;
+        m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_UNORM_PACK32]      = TEX_FORMAT_RGB10A2_UNORM;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_SNORM_PACK32]      = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_USCALED_PACK32]    = TEX_FORMAT_UNKNOWN;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_SSCALED_PACK32]    = TEX_FORMAT_UNKNOWN;
-        m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_UINT_PACK32]       = TEX_FORMAT_UNKNOWN;
+        m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_UINT_PACK32]       = TEX_FORMAT_RGB10A2_UINT;
         m_VkFmtToTexFmtMap[VK_FORMAT_A2B10G10R10_SINT_PACK32]       = TEX_FORMAT_UNKNOWN;
 
         m_VkFmtToTexFmtMap[VK_FORMAT_R16_UNORM]             = TEX_FORMAT_R16_UNORM;

Tests/DiligentCoreAPITest/src/ClearRenderTargetTest.cpp

@@ -491,4 +491,67 @@ TEST(ClearRenderTargetTest, LoadOpClear2)
     TestClearWithLoadOp({0.375f, 0.75, 0.125, 1.0f}, 2);
 }
 
+// TEX_FORMAT_RGB10A2_UNORM has the layout of DXGI_FORMAT_R10G10B10A2_UNORM and GL_RGB10_A2: red in the low 10 bits
+// and alpha in the high 2 bits. Every backend must store a red clear with this layout.
+TEST(ClearRenderTargetTest, RGB10A2BitLayout)
+{
+    GPUTestingEnvironment* pEnv     = GPUTestingEnvironment::GetInstance();
+    IRenderDevice*         pDevice  = pEnv->GetDevice();
+    IDeviceContext*        pContext = pEnv->GetDeviceContext();
+
+    if ((pDevice->GetTextureFormatInfoExt(TEX_FORMAT_RGB10A2_UNORM).BindFlags & BIND_RENDER_TARGET) == 0)
+    {
+        GTEST_SKIP() << "RGB10A2 render targets are not supported by this device";
+    }
+
+    GPUTestingEnvironment::ScopedReset EnvironmentAutoReset;
+
+    TextureDesc TexDesc;
+    TexDesc.Name      = "ClearRenderTargetTest.RGB10A2BitLayout";
+    TexDesc.Type      = RESOURCE_DIM_TEX_2D;
+    TexDesc.Width     = 4;
+    TexDesc.Height    = 4;
+    TexDesc.Format    = TEX_FORMAT_RGB10A2_UNORM;
+    TexDesc.BindFlags = BIND_RENDER_TARGET;
+    RefCntAutoPtr<ITexture> pTex;
+    pDevice->CreateTexture(TexDesc, nullptr, &pTex);
+    ASSERT_NE(pTex, nullptr);
+
+    TexDesc.Name           = "ClearRenderTargetTest.RGB10A2BitLayout staging";
+    TexDesc.Usage          = USAGE_STAGING;
+    TexDesc.BindFlags      = BIND_NONE;
+    TexDesc.CPUAccessFlags = CPU_ACCESS_READ;
+    RefCntAutoPtr<ITexture> pStagingTex;
+    pDevice->CreateTexture(TexDesc, nullptr, &pStagingTex);
+    ASSERT_NE(pStagingTex, nullptr);
+
+    ITextureView* pRTVs[] = {pTex->GetDefaultView(TEXTURE_VIEW_RENDER_TARGET)};
+    pContext->SetRenderTargets(1, pRTVs, nullptr, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    constexpr float Red[] = {1, 0, 0, 1};
+    pContext->ClearRenderTarget(pRTVs[0], Red, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    pContext->SetRenderTargets(0, nullptr, nullptr, RESOURCE_STATE_TRANSITION_MODE_NONE);
+
+    pContext->CopyTexture(CopyTextureAttribs{pTex, RESOURCE_STATE_TRANSITION_MODE_TRANSITION, pStagingTex, RESOURCE_STATE_TRANSITION_MODE_TRANSITION});
+    pContext->WaitForIdle();
+
+    // Some Intel D3D11 drivers return null with DO_NOT_WAIT even after WaitForIdle().
+    const MAP_FLAGS          MapFlags = pDevice->GetDeviceInfo().Type == RENDER_DEVICE_TYPE_D3D11 ? MAP_FLAG_NONE : MAP_FLAG_DO_NOT_WAIT;
+    MappedTextureSubresource MappedData;
+    pContext->MapTextureSubresource(pStagingTex, 0, 0, MAP_READ, MapFlags, nullptr, MappedData);
+    ASSERT_NE(MappedData.pData, nullptr);
+
+    constexpr Uint32 RedTexel = (3u << 30) | 0x3FFu;
+    for (Uint32 Row = 0; Row < TexDesc.Height; ++Row)
+    {
+        const Uint8* pRow = static_cast<const Uint8*>(MappedData.pData) + Row * MappedData.Stride;
+        for (Uint32 Col = 0; Col < TexDesc.Width; ++Col)
+        {
+            Uint32 Texel = 0;
+            memcpy(&Texel, pRow + Col * sizeof(Texel), sizeof(Texel));
+            EXPECT_EQ(Texel, RedTexel) << "Texel (" << Col << ", " << Row << "): expected red in bits 0-9 and alpha in bits 30-31";
+        }
+    }
+    pContext->UnmapTextureSubresource(pStagingTex, 0, 0);
+}
+
 } // namespace