diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1
index 0c2431cc7e74..48b81498728c 100644
--- a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1
+++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1
@@ -23,6 +23,7 @@ function Invoke-LoggedCommand
         [string] $ExecutePath,
         [switch] $GroupOutput,
         [int[]] $AllowedExitCodes = @(0),
+        [switch] $DoNotExitOnFailedExitCode,
         [scriptblock] $OutputProcessor
     )
 
@@ -51,17 +52,11 @@ function Invoke-LoggedCommand
         LogGroupEnd
       }
 
-      if($LastExitCode -notin $AllowedExitCodes)
+      if($LASTEXITCODE -notin $AllowedExitCodes)
       {
           LogError "Command failed to execute ($duration): $Command`n"
-
-          # This fix reproduces behavior that existed before 
-          # https://github.com/Azure/azure-sdk-tools/pull/12235 
-          # Before that change, if a command failed Write-Error was always 
-          # invoked in the failure case. Today, LogError only does Write-Error
-          # when running locally (not in a CI environment)
-          if ((Test-SupportsDevOpsLogging) -or (Test-SupportsGitHubLogging)) {
-              Write-Error "Command failed to execute ($duration): $Command`n"
+          if (!$DoNotExitOnFailedExitCode) {
+              exit $LASTEXITCODE
           }
       }
       else {
diff --git a/eng/common/scripts/logging.ps1 b/eng/common/scripts/logging.ps1
index 4700a5716453..c57bd0a51972 100644
--- a/eng/common/scripts/logging.ps1
+++ b/eng/common/scripts/logging.ps1
@@ -38,7 +38,7 @@ function LogWarning {
     Write-Host ("::warning::$args" -replace "`n", "%0D%0A")
   }
   else {
-    Write-Warning "$args"
+    Write-Host "$args" -ForegroundColor Yellow
   }
 }
 
@@ -59,7 +59,7 @@ function LogErrorForFile($file, $errorString)
     Write-Host ("::error file=$file,line=1,col=1::$errorString" -replace "`n", "%0D%0A")
   }
   else {
-    Write-Error "[Error in file $file]$errorString"
+    Write-Host "[Error in file $file]$errorString" -ForegroundColor Red
   }
 }
 
@@ -71,7 +71,7 @@ function LogError {
     Write-Host ("::error::$args" -replace "`n", "%0D%0A")
   }
   else {
-    Write-Error "$args"
+    Write-Host "$args" -ForegroundColor Red
   }
 }