diff --git a/TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset b/TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset
index a7ba7ccdc..2aea15a22 100644
--- a/TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset
+++ b/TestProjects/PerceptionHDRP/Assets/SemanticSegmentationLabelingConfiguration.asset
@@ -21,3 +21,4 @@ MonoBehaviour:
     color: {r: 0, g: 1, b: 0.16973758, a: 1}
   - label: Terrain
     color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
+  skyColor: {r: 0, g: 0, b: 0, a: 1}
diff --git a/TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset b/TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset
index a7ba7ccdc..2aea15a22 100644
--- a/TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset
+++ b/TestProjects/PerceptionURP/Assets/SemanticSegmentationLabelingConfiguration.asset
@@ -21,3 +21,4 @@ MonoBehaviour:
     color: {r: 0, g: 1, b: 0.16973758, a: 1}
   - label: Terrain
     color: {r: 0.8207547, g: 0, b: 0.6646676, a: 1}
+  skyColor: {r: 0, g: 0, b: 0, a: 1}
diff --git a/TestProjects/PerceptionURP/Packages/packages-lock.json b/TestProjects/PerceptionURP/Packages/packages-lock.json
index 8f8d5d7c4..320861758 100644
--- a/TestProjects/PerceptionURP/Packages/packages-lock.json
+++ b/TestProjects/PerceptionURP/Packages/packages-lock.json
@@ -81,7 +81,7 @@
         "com.unity.collections": "0.9.0-preview.6",
         "com.unity.nuget.newtonsoft-json": "1.1.2",
         "com.unity.render-pipelines.core": "7.1.6",
-        "com.unity.simulation.capture": "0.0.10-preview.19",
+        "com.unity.simulation.capture": "0.0.10-preview.20",
         "com.unity.simulation.client": "0.0.10-preview.10",
         "com.unity.simulation.core": "0.0.10-preview.22"
       }
@@ -115,7 +115,7 @@
       "url": "https://packages.unity.com"
     },
     "com.unity.simulation.capture": {
-      "version": "0.0.10-preview.19",
+      "version": "0.0.10-preview.20",
       "depth": 1,
       "source": "registry",
       "dependencies": {
diff --git a/com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs b/com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs
index a2583921b..8897024c9 100644
--- a/com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs
+++ b/com.unity.perception/Editor/GroundTruth/IdLabelConfigEditor.cs
@@ -37,6 +37,8 @@ protected override void InitUiExtended()
 
             m_StartingIdEnumField.SetEnabled(AutoAssign);
 
+            m_SkyColorUi.style.display = DisplayStyle.None;
+
             AutoAssignIdsIfNeeded();
             m_MoveDownButton.clicked += MoveSelectedItemDown;
             m_MoveUpButton.clicked += MoveSelectedItemUp;
diff --git a/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs b/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs
index 762c00e14..93d2a6f78 100644
--- a/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs
+++ b/com.unity.perception/Editor/GroundTruth/LabelConfigEditor.cs
@@ -41,9 +41,13 @@ abstract class LabelConfigEditor<T> : Editor where T : ILabelEntry
         protected Button m_MoveDownButton;
         protected VisualElement m_MoveButtons;
         protected VisualElement m_IdSpecificUi;
+        protected VisualElement m_SkyColorUi;
         protected EnumField m_StartingIdEnumField;
         protected Toggle m_AutoIdToggle;
 
+        protected ColorField m_SkyColorField;
+        protected Label m_SkyHexLabel;
+
 
         public void OnEnable()
         {
@@ -114,6 +118,9 @@ void InitUi()
             m_StartingIdEnumField = m_Root.Q<EnumField>("starting-id-dropdown");
             m_AutoIdToggle = m_Root.Q<Toggle>("auto-id-toggle");
             m_IdSpecificUi = m_Root.Q<VisualElement>("id-specific-ui");
+            m_SkyColorUi = m_Root.Q<VisualElement>("sky-color-ui");
+            m_SkyColorField = m_Root.Q<ColorField>("sky-color-value");
+            m_SkyHexLabel = m_Root.Q<Label>("sky-color-hex");
 
             m_SaveButton.SetEnabled(false);
 
diff --git a/com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs b/com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs
index 4fbebfaa0..d1fcc47bd 100644
--- a/com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs
+++ b/com.unity.perception/Editor/GroundTruth/SemanticSegmentationLabelConfigEditor.cs
@@ -15,6 +15,16 @@ protected override void InitUiExtended()
         {
             m_MoveButtons.style.display = DisplayStyle.None;
             m_IdSpecificUi.style.display = DisplayStyle.None;
+
+            var skyColorProperty = serializedObject.FindProperty(nameof(SemanticSegmentationLabelConfig.skyColor));
+            m_SkyColorField.BindProperty(skyColorProperty);
+            m_SkyColorField.RegisterValueChangedCallback(e => UpdateSkyHexLabel(e.newValue));
+            UpdateSkyHexLabel(skyColorProperty.colorValue);
+        }
+
+        private void UpdateSkyHexLabel(Color colorValue)
+        {
+            m_SkyHexLabel.text = "#" + ColorUtility.ToHtmlStringRGBA(colorValue);
         }
 
         public override void PostRemoveOperations()
diff --git a/com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml b/com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml
index 6b7e5addc..7ef3d081f 100644
--- a/com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml
+++ b/com.unity.perception/Editor/GroundTruth/Uxml/ColoredLabelElementInLabelConfig.uxml
@@ -1,12 +1,12 @@
 <UXML xmlns="UnityEngine.UIElements" xmlns:editor="UnityEditor.UIElements">
     <VisualElement class="added-label" style="padding-top: 3px;">
         <Button name="remove-button" class="labeling__remove-item-button"/>
+        <TextField name="label-value" class="labeling__added-label-value"/>
         <VisualElement class="generic-hover"
                        style="min-width : 137px; flex-direction: row; padding: 3px 0 3px 6px; margin-left: 3px; margin-right: 3px; border-width: 1px; border-color: #555555; border-radius: 4px;">
             <editor:ColorField name="label-color-value" style="min-width : 60px; max-width: 60px; align-self:center; margin:2px"/>
             <Label name="label-color-hex" style="font-size: 11; min-width : 60px; max-width: 60px; align-self:center;"/>
         </VisualElement>
-        <TextField name="label-value" class="labeling__added-label-value"/>
         <VisualElement style="min-width:20px; flex-direction: row; display:none">
             <Button name="move-up-button" class="move-label-in-config-button move-up" style="margin-right:-2px"/>
             <Button name="move-down-button" class="move-label-in-config-button move-down"/>
diff --git a/com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml b/com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml
index 7a168acea..f49727d43 100644
--- a/com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml
+++ b/com.unity.perception/Editor/GroundTruth/Uxml/LabelConfig_Main.uxml
@@ -2,12 +2,26 @@
     <VisualElement class="outer-container" name="outer-container">
         <Style src="../Uss/Styles.uss"/>
         <VisualElement class="inner-container" name="id-specific-ui">
+            <Label text="Options" name="options-title" class="title-label"/>
             <Toggle name="auto-id-toggle" text="Auto Assign IDs" style="margin:0" binding-path="autoAssignIds"/>
             <VisualElement style="flex-direction: row; flex-grow: 1;">
                 <editor:EnumField label="Starting ID" name="starting-id-dropdown" binding-path="startingLabelId"
                                   style="flex-grow:1; margin:0"/>
             </VisualElement>
         </VisualElement>
+        <VisualElement class="inner-container" name="sky-color-ui">
+            <Label text="Options" name="options-title" class="title-label"/>
+            <VisualElement style="flex-direction: row; flex-grow: 1;">
+                <Label text="Sky Color" name="added-labels-title"
+                       style="flex-grow: 1; font-size: 11; min-width : 60px; align-self:center;"/>
+                <VisualElement class="generic-hover"
+                               style="min-width : 137px; flex-direction: row; padding: 3px 5px 3px 5px; margin-left: 3px; margin-right: 3px; border-width: 1px; border-color: #666666; border-radius: 4px;">
+                    <editor:ColorField name="sky-color-value" style="min-width : 60px; max-width: 60px; align-self:center;"/>
+                    <Label name="sky-color-hex"
+                           style="font-size: 11; min-width : 60px; max-width: 60px; align-self:center; margin: 2px"/>
+                </VisualElement>
+            </VisualElement>
+        </VisualElement>
         <VisualElement name="added-labels" class="inner-container" style="margin-top:5px">
             <Label text="Added Labels" name="added-labels-title" class="title-label"/>
             <ListView name="labels-listview" class="labeling__label-listview" style="margin-top: 5px;"/>
diff --git a/com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs b/com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs
index c07a75c23..15f1094f1 100644
--- a/com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs
+++ b/com.unity.perception/Runtime/GroundTruth/Labeling/SemanticSegmentationLabelConfig.cs
@@ -20,6 +20,12 @@ public class SemanticSegmentationLabelConfig : LabelConfig<SemanticSegmentationL
             Color.yellow,
             Color.gray
         };
+
+
+        /// <summary>
+        /// The color to use for the sky in semantic segmentation images
+        /// </summary>
+        public Color skyColor = Color.black;
     }
 
     /// <summary>
diff --git a/com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs b/com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs
index 320cb9cb9..21dce9519 100644
--- a/com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs
+++ b/com.unity.perception/Runtime/GroundTruth/RenderPasses/CrossPipelinePasses/SemanticSegmentationCrossPipelinePass.cs
@@ -56,7 +56,7 @@ protected override void ExecutePass(
 
             s_LastFrameExecuted = Time.frameCount;
             var renderList = CreateRendererListDesc(camera, cullingResult, "FirstPass", 0, m_OverrideMaterial, -1);
-            cmd.ClearRenderTarget(true, true, Color.black);
+            cmd.ClearRenderTarget(true, true, m_LabelConfig.skyColor);
             DrawRendererList(renderContext, cmd, RendererList.Create(renderList));
         }
 
diff --git a/com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs b/com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs
index 61643b133..74582b7c3 100644
--- a/com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs
+++ b/com.unity.perception/Tests/Runtime/GroundTruthTests/SegmentationGroundTruthTests.cs
@@ -63,6 +63,7 @@ public class SegmentationPassTests : GroundTruthTestBase
     {
         static readonly Color32 k_SemanticPixelValue = new Color32(10, 20, 30, Byte.MaxValue);
         private static readonly Color32 k_InstanceSegmentationPixelValue = new Color32(255,0,0, 255);
+        private static readonly Color32 k_SkyValue = new Color32(10, 20, 30, 40);
 
         public enum SegmentationKind
         {
@@ -92,7 +93,6 @@ void OnSegmentationImageReceived<T>(int frameCount, NativeArray<T> data, RenderT
             switch (segmentationKind)
             {
                 case SegmentationKind.Instance:
-                    //expectedPixelValue = new Color32(0, 74, 255, 255);
                     expectedPixelValue = k_InstanceSegmentationPixelValue;
                     cameraObject = SetupCameraInstanceSegmentation(OnSegmentationImageReceived);
                     break;
@@ -247,7 +247,7 @@ void OnSegmentationImageReceived(NativeArray<Color32> data)
                 CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
             }
 
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
 
             AddTestObjectForCleanup(TestHelper.CreateLabeledPlane(label: "non-matching"));
             yield return null;
@@ -267,7 +267,7 @@ void OnSegmentationImageReceived(NativeArray<Color32> data)
                 CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
             }
 
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), false, k_SkyValue);
 
             var gameObject = TestHelper.CreateLabeledPlane();
             gameObject.GetComponent<Labeling>().enabled = false;
@@ -301,17 +301,17 @@ void OnSegmentationImageReceived(NativeArray<Color32> data)
         }
 
         [UnityTest]
-        public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesBlack([Values(false, true)] bool showVisualizations)
+        public IEnumerator SemanticSegmentationPass_WithEmptyFrame_ProducesSky([Values(false, true)] bool showVisualizations)
         {
             int timesSegmentationImageReceived = 0;
-            var expectedPixelValue = new Color32(0, 0, 0, 255);
+            var expectedPixelValue = k_SkyValue;
             void OnSegmentationImageReceived(NativeArray<Color32> data)
             {
                 timesSegmentationImageReceived++;
                 CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
             }
 
-            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations);
+            var cameraObject = SetupCameraSemanticSegmentation(a => OnSegmentationImageReceived(a.data), showVisualizations, expectedPixelValue);
 
             //TestHelper.LoadAndStartRenderDocCapture(out var gameView);
             yield return null;
@@ -333,6 +333,26 @@ void OnSegmentationImageReceived(NativeArray<Color32> data)
             Assert.AreEqual(1, timesSegmentationImageReceived);
         }
 
+        [UnityTest]
+        public IEnumerator SemanticSegmentationPass_WithNoObjects_ProducesSky()
+        {
+            int timesSegmentationImageReceived = 0;
+            var expectedPixelValue = k_SkyValue;
+            void OnSegmentationImageReceived(NativeArray<Color32> data)
+            {
+                timesSegmentationImageReceived++;
+                CollectionAssert.AreEqual(Enumerable.Repeat(expectedPixelValue, data.Length), data.ToArray());
+            }
+
+            var cameraObject = SetupCameraSemanticSegmentation(
+                a => OnSegmentationImageReceived(a.data), false, expectedPixelValue);
+
+            yield return null;
+            //destroy the object to force all pending segmented image readbacks to finish and events to be fired.
+            DestroyTestObject(cameraObject);
+            Assert.AreEqual(1, timesSegmentationImageReceived);
+        }
+
         [UnityTest]
         public IEnumerator SemanticSegmentationPass_WithTextureOverride_RendersToOverride([Values(true, false)] bool showVisualizations)
         {
@@ -524,7 +544,7 @@ GameObject SetupCameraInstanceSegmentation(Action<int, NativeArray<Color32>, Ren
             return cameraObject;
         }
 
-        GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations)
+        GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.ImageReadbackEventArgs> onSegmentationImageReceived, bool showVisualizations, Color? backgroundColor = null)
         {
             var cameraObject = SetupCamera(out var perceptionCamera, showVisualizations);
             var labelConfig = ScriptableObject.CreateInstance<SemanticSegmentationLabelConfig>();
@@ -536,6 +556,10 @@ GameObject SetupCameraSemanticSegmentation(Action<SemanticSegmentationLabeler.Im
                     color = k_SemanticPixelValue
                 }
             });
+            if (backgroundColor != null)
+            {
+                labelConfig.skyColor = backgroundColor.Value;
+            }
             var semanticSegmentationLabeler = new SemanticSegmentationLabeler(labelConfig);
             semanticSegmentationLabeler.imageReadback += onSegmentationImageReceived;
             perceptionCamera.AddLabeler(semanticSegmentationLabeler);