add stress test (pomerium#609)

cmd/root.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+
+	stress_cmd "github.com/pomerium/ingress-controller/internal/stress/cmd"
 )
 
 // RootCommand generates default secrets
@@ -18,6 +20,7 @@ func RootCommand() (*cobra.Command, error) {
 		"gen-secrets": GenSecretsCommand,
 		"controller":  ControllerCommand,
 		"all-in-one":  AllInOneCommand,
+		"stress-test": stress_cmd.Command,
 	} {
 		cmd, err := fn()
 		if err != nil {

config/stress-test/config.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stress-test
+data:
+  # how many ingresses to create
+  ingress-count: "100"
+  # what is the domain name to use for the ingresses
+  ingress-domain: ""
+  # how long to wait for the ingress to be ready.
+  # this may be proportional to the number of ingresses
+  # the test would crash and start from scratch if the readiness timeout is not long enough
+  readiness-timeout: "5m"

config/stress-test/deployment.yaml

@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: stress-test
+spec:
+  replicas: 1
+  template:
+    spec:
+      serviceAccountName: pomerium-stress-test
+      containers:
+      - name: stress-test
+        args:
+          - "stress-test"
+        image: pomerium/ingress-controller:main
+        imagePullPolicy: Always
+        resources:
+          limits:
+            memory: "256Mi"
+            cpu: "500m"
+        env:
+          - name: SERVICE_NAME
+            value: "stress-test-echo"
+          - name: SERVICE_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+          - name: SERVICE_PORT_NAMES
+            value: "echo1,echo2"
+          - name: CONTAINER_PORT_NUMBERS
+            value: "8081,8082"
+          - name: INGRESS_CLASS
+            value: "pomerium"
+          - name: INGRESS_DOMAIN
+            valueFrom:
+              configMapKeyRef:
+                optional: false
+                name: stress-test
+                key: ingress-domain
+          - name: INGRESS_COUNT
+            valueFrom:
+              configMapKeyRef:
+                optional: false
+                name: stress-test
+                key: ingress-count
+          - name: READINESS_TIMEOUT
+            valueFrom:
+              configMapKeyRef:
+                optional: false
+                name: stress-test
+                key: readiness-timeout
+        ports:
+        - containerPort: 8081
+          name: echo1
+        - containerPort: 8082
+          name: echo2

config/stress-test/kustomization.yaml

@@ -0,0 +1,9 @@
+namespace: pomerium-stress-test
+commonLabels:
+  app.kubernetes.io/name: pomerium-stress-test
+resources:
+  - namespace.yaml
+  - ./rbac
+  - config.yaml
+  - deployment.yaml
+  - service.yaml

config/stress-test/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pomerium-stress-test

config/stress-test/rbac/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - role.yaml
+  - role_binding.yaml
+  - service_account.yaml

config/stress-test/rbac/role.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pomerium-stress-test
+rules:
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete

config/stress-test/rbac/role_binding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pomerium-stress-test
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pomerium-stress-test
+subjects:
+  - kind: ServiceAccount
+    name: pomerium-stress-test

config/stress-test/rbac/service_account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pomerium-stress-test

config/stress-test/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: stress-test-echo
+spec:
+  ports:
+  - port: 8081
+    name: echo1
+    targetPort: echo1
+  - port: 8082
+    name: echo2
+    targetPort: echo2

go.mod

@@ -3,6 +3,7 @@ module github.com/pomerium/ingress-controller
 go 1.20
 
 require (
+	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/client9/misspell v0.3.4
 	github.com/envoyproxy/go-control-plane v0.11.0
 	github.com/go-logr/logr v1.2.4
@@ -17,7 +18,9 @@ require (
 	github.com/iancoleman/strcase v0.2.0
 	github.com/martinlindhe/base36 v1.1.1
 	github.com/open-policy-agent/opa v0.51.0
+	github.com/pomerium/csrf v1.7.0
 	github.com/pomerium/pomerium v0.20.1-0.20230421153948-65e0fcb667a6
+	github.com/rs/zerolog v1.29.0
 	github.com/sergi/go-diff v1.3.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
@@ -96,7 +99,6 @@ require (
 	github.com/breml/errchkjson v0.3.1 // indirect
 	github.com/butuzov/ireturn v0.1.1 // indirect
 	github.com/caddyserver/certmagic v0.17.2 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/charithe/durationcheck v0.0.10 // indirect
@@ -239,7 +241,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polyfloyd/go-errorlint v1.4.0 // indirect
-	github.com/pomerium/csrf v1.7.0 // indirect
 	github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 // indirect
 	github.com/pomerium/webauthn v0.0.0-20221118023040-00a9c430578b // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -255,7 +256,6 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/cors v1.8.3 // indirect
-	github.com/rs/zerolog v1.29.0 // indirect
 	github.com/ryancurrah/gomodguard v1.3.0 // indirect
 	github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect
 	github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect

internal/stress/cmd/command.go

@@ -0,0 +1,173 @@
+// Package cmd provides the stress test command
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+
+	"github.com/pomerium/ingress-controller/internal/stress"
+)
+
+// Command returns the stress test command
+func Command() (*cobra.Command, error) {
+	return &cobra.Command{
+		Use:   "stress-test",
+		Short: "stress test the ingress controller",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			setupLogger()
+			return run(withInterruptSignal(context.Background()))
+		},
+	}, nil
+}
+
+func setupLogger() {
+	logger := zerolog.New(os.Stdout)
+	zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	zerolog.DefaultContextLogger = &logger
+}
+
+func testConfigFromEnv() (*stress.IngressLoadTestConfig, error) {
+	svcName := os.Getenv("SERVICE_NAME")
+	if svcName == "" {
+		return nil, fmt.Errorf("SERVICE_NAME environment variable not set")
+	}
+
+	svcNamespace := os.Getenv("SERVICE_NAMESPACE")
+	if svcNamespace == "" {
+		return nil, fmt.Errorf("SERVICE_NAMESPACE environment variable not set")
+	}
+
+	servicePortNames := strings.Split(os.Getenv("SERVICE_PORT_NAMES"), ",")
+	if len(servicePortNames) != 2 {
+		return nil, fmt.Errorf("SERVICE_PORT_NAMES environment variable must have exacly two comma separated values")
+	}
+
+	domain := os.Getenv("INGRESS_DOMAIN")
+	if domain == "" {
+		return nil, fmt.Errorf("INGRESS_DOMAIN environment variable not set")
+	}
+
+	ingressCount, err := strconv.Atoi(os.Getenv("INGRESS_COUNT"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse INGRESS_COUNT: %w", err)
+	}
+
+	ingressClass := os.Getenv("INGRESS_CLASS")
+	if ingressClass == "" {
+		return nil, fmt.Errorf("INGRESS_CLASS environment variable not set")
+	}
+
+	readinessTimeout, err := time.ParseDuration(os.Getenv("READINESS_TIMEOUT"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse READINESS_TIMEOUT: %w", err)
+	}
+
+	return &stress.IngressLoadTestConfig{
+		ReadinessTimeout: readinessTimeout,
+		IngressClass:     ingressClass,
+		IngressCount:     ingressCount,
+		ServicePortNames: servicePortNames,
+		ServiceName:      types.NamespacedName{Name: svcName, Namespace: svcNamespace},
+		Domain:           domain,
+	}, nil
+}
+
+func getKubeClient() (*kubernetes.Clientset, error) {
+	var kubeconfig *rest.Config
+	kubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config")
+	if _, err := os.Stat(kubeconfigPath); err == nil {
+		kubeconfig, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build kubeconfig from %s: %w", kubeconfigPath, err)
+		}
+	} else {
+		kubeconfig, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get in cluster kubeconfig: %w", err)
+		}
+	}
+
+	client, err := kubernetes.NewForConfig(kubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create kubernetes client: %w", err)
+	}
+
+	return client, nil
+}
+
+func run(ctx context.Context) error {
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.Go(func() error {
+		return runTest(ctx)
+	})
+	eg.Go(func() error {
+		return runEcho(ctx)
+	})
+	return eg.Wait()
+}
+
+func runEcho(ctx context.Context) error {
+	ports := strings.Split(os.Getenv("CONTAINER_PORT_NUMBERS"), ",")
+	if len(ports) != 2 {
+		return fmt.Errorf("CONTAINER_PORT_NUMBERS should have exactly two comma separated values")
+	}
+	eg, ctx := errgroup.WithContext(ctx)
+	for _, port := range ports {
+		port := port
+		eg.Go(func() error {
+			n, err := strconv.Atoi(port)
+			if err != nil {
+				return fmt.Errorf("failed to parse port number: %w", err)
+			}
+			return stress.RunHTTPEchoServer(ctx, fmt.Sprintf(":%d", n))
+		})
+	}
+	return eg.Wait()
+}
+
+func runTest(ctx context.Context) error {
+	cfg, err := testConfigFromEnv()
+	if err != nil {
+		return err
+	}
+
+	client, err := getKubeClient()
+	if err != nil {
+		return err
+	}
+
+	cfg.Client = client
+	srv := stress.IngressLoadTest{IngressLoadTestConfig: *cfg}
+	err = srv.Run(ctx)
+	if err != nil {
+		zerolog.Ctx(ctx).Error().Err(err).Msg("error running stress test...")
+	}
+	return err
+}
+
+func withInterruptSignal(ctx context.Context) context.Context {
+	ctx, cancel := context.WithCancel(ctx)
+	go func() {
+		ch := make(chan os.Signal, 2)
+		signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+		<-ch
+		cancel()
+	}()
+	return ctx
+}