diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go
index 79dbf7e..7ad620d 100644
--- a/pkg/replacer/replacer.go
+++ b/pkg/replacer/replacer.go
@@ -61,6 +61,8 @@ type Replacer struct {
 
 // NewGitHubActionsReplacer creates a new replacer for GitHub actions
 func NewGitHubActionsReplacer(cfg *config.Config) *Replacer {
+	cfg = config.MergeUserConfig(cfg)
+
 	return &Replacer{
 		cfg:    *cfg,
 		parser: actions.New(),
@@ -70,6 +72,8 @@ func NewGitHubActionsReplacer(cfg *config.Config) *Replacer {
 
 // NewContainerImagesReplacer creates a new replacer for container images
 func NewContainerImagesReplacer(cfg *config.Config) *Replacer {
+	cfg = config.MergeUserConfig(cfg)
+
 	return &Replacer{
 		cfg:    *cfg,
 		parser: image.New(),
diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go
index 6164a5c..f50c87b 100644
--- a/pkg/replacer/replacer_test.go
+++ b/pkg/replacer/replacer_test.go
@@ -196,8 +196,7 @@ func TestReplacer_ParseContainerImageString(t *testing.T) {
 			config := &config.Config{
 				Images: config.Images{
 					ImageFilter: config.ImageFilter{
-						ExcludeImages: []string{"scratch"},
-						ExcludeTags:   []string{"latest"},
+						ExcludeTags: []string{"latest"},
 					},
 				},
 			}
@@ -407,8 +406,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) {
 				},
 				Images: config.Images{
 					ImageFilter: config.ImageFilter{
-						ExcludeImages: []string{"scratch"},
-						ExcludeTags:   []string{"latest"},
+						ExcludeTags: []string{"latest"},
 					},
 				},
 			}
@@ -728,8 +726,7 @@ CMD ["dex", "serve", "/etc/dex/config.docker.yaml"]
 			r := NewContainerImagesReplacer(&config.Config{
 				Images: config.Images{
 					ImageFilter: config.ImageFilter{
-						ExcludeImages: []string{"scratch"},
-						ExcludeTags:   []string{"latest"},
+						ExcludeTags: []string{"latest"},
 					},
 				},
 			})
diff --git a/pkg/utils/config/config.go b/pkg/utils/config/config.go
index eea4162..792b530 100644
--- a/pkg/utils/config/config.go
+++ b/pkg/utils/config/config.go
@@ -22,6 +22,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/osfs"
@@ -109,6 +110,24 @@ func DefaultConfig() *Config {
 	}
 }
 
+// MergeUserConfig merges the user configuration with the default configuration.
+// mostly making sure that we don't try to pin the scratch image
+func MergeUserConfig(userConfig *Config) *Config {
+	if userConfig == nil {
+		return DefaultConfig()
+	}
+
+	if userConfig.Images.ExcludeImages == nil {
+		userConfig.Images.ExcludeImages = []string{"scratch"}
+	}
+
+	if !slices.Contains(userConfig.Images.ExcludeImages, "scratch") {
+		userConfig.Images.ExcludeImages = append(userConfig.Images.ExcludeImages, "scratch")
+	}
+
+	return userConfig
+}
+
 // ParseConfigFileFromFS parses a configuration file from a filesystem.
 func ParseConfigFileFromFS(fs billy.Filesystem, configfile string) (*Config, error) {
 	cfg := DefaultConfig()