Skip to content

[Contour Gateway Provisioner] add imagepullsecret for envoy and contour (fixes #7138) #7141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[Contour Gateway Provisioner] add imagepullsecret for envoy and contour (fixes #7138) #7141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelogs/unreleased/7141-flbla-small.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
add a new flag: --image-pull-secret-name, which allows users to specify a secret in the same namespace as the deployed contour control plane for pulling images from private registries. when set, it's used to pull Envoy and Contour images.
11 changes: 10 additions & 1 deletion cmd/contour/gatewayprovisioner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func registerGatewayProvisioner(app *kingpin.Application) (*kingpin.CmdClause, *
leaderElection: false,
leaderElectionID: "0d879e31.projectcontour.io",
gatewayControllerName: "projectcontour.io/gateway-controller",
imagePullSecret: "",
}

cmd.Flag("contour-image", "The container image used for the managed Contour.").
Expand All @@ -58,6 +59,10 @@ func registerGatewayProvisioner(app *kingpin.Application) (*kingpin.CmdClause, *
Default(provisionerConfig.gatewayControllerName).
StringVar(&provisionerConfig.gatewayControllerName)

cmd.Flag("image-pull-secret-name", "The image pull secret for the managed Envoy and Contour.").
Default(provisionerConfig.imagePullSecret).
StringVar(&provisionerConfig.imagePullSecret)

cmd.Flag("incluster", "Use in cluster configuration.").
Default("true").
BoolVar(&provisionerConfig.inCluster)
Expand Down Expand Up @@ -85,6 +90,10 @@ type gatewayProvisionerConfig struct {
// by the gateway provisioner.
envoyImage string

// imagePullSecret is the name of the image pull secret that will be used
// to pull the Contour and Envoy images.
imagePullSecret string

// metricsBindAddress is the TCP address that the gateway provisioner should bind to for
// serving prometheus metrics. It can be set to "0" to disable the metrics serving.
metricsBindAddress string
Expand Down Expand Up @@ -171,7 +180,7 @@ func createManager(restConfig *rest.Config, provisionerConfig *gatewayProvisione
if _, err := controller.NewGatewayClassController(mgr, provisionerConfig.gatewayControllerName); err != nil {
return nil, fmt.Errorf("failed to create gatewayclass controller: %w", err)
}
if _, err := controller.NewGatewayController(mgr, provisionerConfig.gatewayControllerName, provisionerConfig.contourImage, provisionerConfig.envoyImage); err != nil {
if _, err := controller.NewGatewayController(mgr, provisionerConfig.gatewayControllerName, provisionerConfig.contourImage, provisionerConfig.envoyImage, provisionerConfig.imagePullSecret); err != nil {
return nil, fmt.Errorf("failed to create gateway controller: %w", err)
}
return mgr, nil
Expand Down
8 changes: 5 additions & 3 deletions internal/provisioner/controller/gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,17 @@ type gatewayReconciler struct {
gatewayController gatewayapi_v1.GatewayController
contourImage string
envoyImage string
imagePullSecret string
client client.Client
log logr.Logger
}

func NewGatewayController(mgr manager.Manager, gatewayController, contourImage, envoyImage string) (controller.Controller, error) {
func NewGatewayController(mgr manager.Manager, gatewayController, contourImage, envoyImage, imagePullSecret string) (controller.Controller, error) {
r := &gatewayReconciler{
gatewayController: gatewayapi_v1.GatewayController(gatewayController),
contourImage: contourImage,
envoyImage: envoyImage,
imagePullSecret: imagePullSecret,
client: mgr.GetClient(),
log: ctrl.Log.WithName("gateway-controller"),
}
Expand Down Expand Up @@ -432,8 +434,8 @@ func (r *gatewayReconciler) ensureContour(ctx context.Context, contour *model.Co

handleResult("contour config", contourconfig.EnsureContourConfig(ctx, r.client, contour))
handleResult("xDS TLS secrets", secret.EnsureXDSSecrets(ctx, r.client, contour, r.contourImage))
handleResult("deployment", deployment.EnsureDeployment(ctx, r.client, contour, r.contourImage))
handleResult("envoy data plane", dataplane.EnsureDataPlane(ctx, r.client, contour, r.contourImage, r.envoyImage))
handleResult("deployment", deployment.EnsureDeployment(ctx, r.client, contour, r.contourImage, r.imagePullSecret))
handleResult("envoy data plane", dataplane.EnsureDataPlane(ctx, r.client, contour, r.contourImage, r.envoyImage, r.imagePullSecret))
handleResult("contour service", service.EnsureContourService(ctx, r.client, contour))

switch contour.Spec.NetworkPublishing.Envoy.Type {
Expand Down
13 changes: 7 additions & 6 deletions internal/provisioner/equality/equality_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ import (
)

var (
testName = "test"
testNs = testName + "-ns"
testImage = "test-image:main"
cntr = &model.Contour{
testName = "test"
testNs = testName + "-ns"
testImage = "test-image:main"
testImagePullSecret = ""
cntr = &model.Contour{
ObjectMeta: meta_v1.ObjectMeta{
Name: testName,
Namespace: testNs,
Expand Down Expand Up @@ -122,7 +123,7 @@ func TestDaemonSetConfigChanged(t *testing.T) {

for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
original := dataplane.DesiredDaemonSet(cntr, testImage, testImage)
original := dataplane.DesiredDaemonSet(cntr, testImage, testImage, testImagePullSecret)

mutated := original.DeepCopy()
tc.mutate(mutated)
Expand Down Expand Up @@ -218,7 +219,7 @@ func TestDeploymentConfigChanged(t *testing.T) {
}

for _, tc := range testCases {
original := deployment.DesiredDeployment(cntr, testImage)
original := deployment.DesiredDeployment(cntr, testImage, testImagePullSecret)
mutated := original.DeepCopy()
tc.mutate(mutated)
if updated, changed := equality.DeploymentConfigChanged(original, mutated); changed != tc.expect {
Expand Down
26 changes: 21 additions & 5 deletions internal/provisioner/objects/dataplane/dataplane.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,11 @@ var defContainerResources = core_v1.ResourceRequirements{
}

// EnsureDataPlane ensures an Envoy data plane (daemonset or deployment) exists for the given contour.
func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Contour, contourImage, envoyImage string) error {
func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Contour, contourImage, envoyImage, imagePullSecret string) error {
switch contour.Spec.EnvoyWorkloadType {
// If a Deployment was specified, provision a Deployment.
case model.WorkloadTypeDeployment:
desired := desiredDeployment(contour, contourImage, envoyImage)
desired := desiredDeployment(contour, contourImage, envoyImage, imagePullSecret)

updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.Deployment) error {
differ := equality.DeploymentSelectorsDiffer(current, desired)
Expand All @@ -94,7 +94,7 @@ func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Cont

// The default workload type is a DaemonSet.
default:
desired := DesiredDaemonSet(contour, contourImage, envoyImage)
desired := DesiredDaemonSet(contour, contourImage, envoyImage, imagePullSecret)

updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.DaemonSet) error {
differ := equality.DaemonSetSelectorsDiffer(current, desired)
Expand Down Expand Up @@ -335,7 +335,7 @@ func desiredContainers(contour *model.Contour, contourImage, envoyImage string)
// DesiredDaemonSet returns the desired DaemonSet for the provided contour using
// contourImage as the shutdown-manager/envoy-initconfig container images and
// envoyImage as Envoy's container image.
func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *apps_v1.DaemonSet {
func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage, imagePullSecret string) *apps_v1.DaemonSet {
initContainers, containers := desiredContainers(contour, contourImage, envoyImage)

ds := &apps_v1.DaemonSet{
Expand Down Expand Up @@ -403,10 +403,18 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
ds.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Envoy.Tolerations
}

if imagePullSecret != "" {
ds.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
{
Name: imagePullSecret,
},
}
}

return ds
}

func desiredDeployment(contour *model.Contour, contourImage, envoyImage string) *apps_v1.Deployment {
func desiredDeployment(contour *model.Contour, contourImage, envoyImage, imagePullSecret string) *apps_v1.Deployment {
initContainers, containers := desiredContainers(contour, contourImage, envoyImage)

deployment := &apps_v1.Deployment{
Expand Down Expand Up @@ -488,6 +496,14 @@ func desiredDeployment(contour *model.Contour, contourImage, envoyImage string)
deployment.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Envoy.Tolerations
}

if imagePullSecret != "" {
deployment.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
{
Name: imagePullSecret,
},
}
}

return deployment
}

Expand Down
30 changes: 26 additions & 4 deletions internal/provisioner/objects/dataplane/dataplane_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ func TestDesiredDaemonSet(t *testing.T) {

testContourImage := "ghcr.io/projectcontour/contour:test"
testEnvoyImage := "docker.io/envoyproxy/envoy:test"
testImagePullSecret := ""
testLogLevelArg := "--log-level debug"
testBaseIDArg := "--base-id 1"
testEnvoyMaxHeapSize := "--overload-max-heap=8000000000"
Expand Down Expand Up @@ -343,7 +344,7 @@ func TestDesiredDaemonSet(t *testing.T) {
cntr.Spec.EnvoyMaxHeapSizeBytes = 8000000000
cntr.Spec.EnvoyMaxDownstreamConnections = 42

ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
container := checkDaemonSetHasContainer(t, ds, EnvoyContainerName, true)
checkContainerHasArg(t, container, testLogLevelArg)
checkContainerHasArg(t, container, testBaseIDArg)
Expand Down Expand Up @@ -386,7 +387,8 @@ func TestDesiredDeployment(t *testing.T) {

testContourImage := "ghcr.io/projectcontour/contour:test"
testEnvoyImage := "docker.io/envoyproxy/envoy:test"
deploy := desiredDeployment(cntr, testContourImage, testEnvoyImage)
testImagePullSecret := ""
deploy := desiredDeployment(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
checkDeploymentHasStrategy(t, deploy, cntr.Spec.EnvoyDeploymentStrategy)
checkEnvoyDeploymentHasAffinity(t, deploy, cntr)
}
Expand Down Expand Up @@ -414,7 +416,8 @@ func TestNodePlacementDaemonSet(t *testing.T) {

testContourImage := "ghcr.io/projectcontour/contour:test"
testEnvoyImage := "docker.io/envoyproxy/envoy:test"
ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
testImagePullSecret := ""
ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
checkDaemonSetHasNodeSelector(t, ds, selectors)
checkDaemonSetHasTolerations(t, ds, tolerations)
}
Expand All @@ -436,9 +439,28 @@ func TestEnvoyCustomPorts(t *testing.T) {

testContourImage := "ghcr.io/projectcontour/contour:test"
testEnvoyImage := "docker.io/envoyproxy/envoy:test"
ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
testImagePullSecret := ""
ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
checkContainerHasPort(t, ds, int32(metricPort))

container := checkDaemonSetHasContainer(t, ds, EnvoyContainerName, true)
checkContainerHasReadinessPort(t, container, 8020)
}

func TestDesiredDaemonSetWithImagePullSecret(t *testing.T) {
name := "ds-test"
cntr := model.Default(fmt.Sprintf("%s-ns", name), name)
cntr.Spec.NetworkPublishing.Envoy.Ports = []model.Port{
{Name: "http", ServicePort: 80, ContainerPort: 8080},
}
testContourImage := "ghcr.io/projectcontour/contour:test"
testEnvoyImage := "docker.io/envoyproxy/envoy:test"
testImagePullSecret := "my-secret"

ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)

require.NotNil(t, ds)
require.NotNil(t, ds.Spec.Template.Spec.ImagePullSecrets)
require.Len(t, ds.Spec.Template.Spec.ImagePullSecrets, 1)
require.Equal(t, testImagePullSecret, ds.Spec.Template.Spec.ImagePullSecrets[0].Name)
}
14 changes: 11 additions & 3 deletions internal/provisioner/objects/deployment/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ const (
)

// EnsureDeployment ensures a deployment using image exists for the given contour.
func EnsureDeployment(ctx context.Context, cli client.Client, contour *model.Contour, image string) error {
desired := DesiredDeployment(contour, image)
func EnsureDeployment(ctx context.Context, cli client.Client, contour *model.Contour, image, imagePullSecret string) error {
desired := DesiredDeployment(contour, image, imagePullSecret)

updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.Deployment) error {
differ := equality.DeploymentSelectorsDiffer(current, desired)
Expand Down Expand Up @@ -82,7 +82,7 @@ func EnsureDeploymentDeleted(ctx context.Context, cli client.Client, contour *mo

// DesiredDeployment returns the desired deployment for the provided contour using
// image as Contour's container image.
func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment {
func DesiredDeployment(contour *model.Contour, image, imagePullSecret string) *apps_v1.Deployment {
xdsPort := objects.XDSPort
args := []string{
"serve",
Expand Down Expand Up @@ -276,6 +276,14 @@ func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment
deploy.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Contour.Tolerations
}

if imagePullSecret != "" {
deploy.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets add some more test coverage (here and the dataplane tests) for the case when imagepullsecret is provided

{
Name: imagePullSecret,
},
}
}

return deploy
}

Expand Down
25 changes: 21 additions & 4 deletions internal/provisioner/objects/deployment/deployment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ func TestDesiredDeployment(t *testing.T) {
}

testContourImage := "ghcr.io/projectcontour/contour:test"
deploy := DesiredDeployment(cntr, testContourImage)
testImagePullSecret := ""
deploy := DesiredDeployment(cntr, testContourImage, testImagePullSecret)

container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
checkContainerHasImage(t, container, testContourImage)
Expand Down Expand Up @@ -239,7 +240,7 @@ func TestDesiredDeploymentWhenSettingWatchNamespaces(t *testing.T) {
cntr.Spec.IngressClassName = &icName
// Change the Contour watch namespaces flag
cntr.Spec.WatchNamespaces = tc.namespaces
deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")
container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
arg := fmt.Sprintf("--watch-namespaces=%s", strings.Join(append(model.NamespacesToStrings(tc.namespaces), cntr.Namespace), ","))
checkContainerHasArg(t, container, arg)
Expand Down Expand Up @@ -268,7 +269,7 @@ func TestNodePlacementDeployment(t *testing.T) {
},
}

deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")

checkDeploymentHasNodeSelector(t, deploy, selectors)
checkDeploymentHasTolerations(t, deploy, tolerations)
Expand Down Expand Up @@ -297,7 +298,7 @@ func TestDesiredDeploymentWhenSettingDisabledFeature(t *testing.T) {
cntr.Spec.IngressClassName = &icName
cntr.Spec.DisabledFeatures = tc.disabledFeatures
// Change the Contour watch namespaces flag
deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")
container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
for _, f := range tc.disabledFeatures {
arg := fmt.Sprintf("--disable-feature=%s", string(f))
Expand All @@ -306,3 +307,19 @@ func TestDesiredDeploymentWhenSettingDisabledFeature(t *testing.T) {
})
}
}

func TestDesiredDeploymentWithImagePullSecret(t *testing.T) {
name := "deploy-test"
cntr := model.Default(fmt.Sprintf("%s-ns", name), name)
icName := "test-ic"
cntr.Spec.IngressClassName = &icName

testContourImage := "ghcr.io/projectcontour/contour:test"
testImagePullSecret := "my-secret"
deploy := DesiredDeployment(cntr, testContourImage, testImagePullSecret)

require.NotNil(t, deploy)
require.NotNil(t, deploy.Spec.Template.Spec.ImagePullSecrets)
require.Len(t, deploy.Spec.Template.Spec.ImagePullSecrets, 1)
require.Equal(t, testImagePullSecret, deploy.Spec.Template.Spec.ImagePullSecrets[0].Name)
}
Loading