Skip to content

Commit

Permalink
certs
Browse files Browse the repository at this point in the history
  • Loading branch information
wasaga committed Aug 19, 2021
1 parent 4010c11 commit e9c2302
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 79 deletions.
56 changes: 34 additions & 22 deletions controllers/reconciler.go → controllers/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ package controllers

import (
"context"
"errors"
"fmt"

certmanagerv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -20,12 +21,14 @@ import (
"github.com/pomerium/ingress-controller/model"
)

// ResourceWatcher watches a given resource to update
// Controller watches ingress and related resources for updates and reconciles with pomerium
type Controller struct {
client.Client
PomeriumReconciler
model.Registry
ingressKind string
secretKind string
serviceKind string
}

// PomeriumReconciler updates pomerium configuration based on provided network resources
Expand All @@ -41,10 +44,12 @@ type PomeriumReconciler interface {
// move the current state of the cluster closer to the desired state.
func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
ic, err := fetchIngress(ctx, r.Client, req.NamespacedName)
ic, err := r.fetchIngress(ctx, req.NamespacedName)
if err != nil {
if !apierrors.IsNotFound(err) {
return ctrl.Result{Requeue: true}, fmt.Errorf("fetch ingress and related resources: %w", err)
if !r.isIngressNotFound(err) {
logger.Error(err, "obtaining ingress related resources", "deps",
r.Registry.Deps(model.Key{Kind: r.ingressKind, NamespacedName: req.NamespacedName}))
return ctrl.Result{Requeue: true}, fmt.Errorf("fetch ingress related resources: %w", err)
}
logger.Info("not found")
if err := r.deleteIngress(ctx, req.NamespacedName); err != nil {
Expand Down Expand Up @@ -87,27 +92,27 @@ func (r *Controller) SetupWithManager(mgr ctrl.Manager) error {
}

scheme := mgr.GetScheme()
if err := certmanagerv1.AddToScheme(scheme); err != nil {
return fmt.Errorf("registering cert-manager types: %w", err)
}

gvk, err := apiutil.GVKForObject(&networkingv1.Ingress{}, scheme)
if err != nil {
return fmt.Errorf("cannot get ingress kind: %w", err)
}
r.ingressKind = gvk.Kind

for _, obj := range []client.Object{
&corev1.Service{},
&corev1.Secret{},
&certmanagerv1.Certificate{},
for _, o := range []struct {
client.Object
kind *string
watch bool
}{
{&networkingv1.Ingress{}, &r.ingressKind, false},
{&corev1.Secret{}, &r.secretKind, true},
{&corev1.Service{}, &r.serviceKind, true},
} {
gvk, err = apiutil.GVKForObject(obj, scheme)
gvk, err := apiutil.GVKForObject(o.Object, scheme)
if err != nil {
return fmt.Errorf("cannot get object kind: %w", err)
return fmt.Errorf("cannot get kind: %w", err)
}
*o.kind = gvk.Kind

if !o.watch {
continue
}

if err := c.Watch(
&source.Kind{Type: obj},
&source.Kind{Type: o.Object},
handler.EnqueueRequestsFromMapFunc(r.getDependantIngressFn(gvk.Kind))); err != nil {
return err
}
Expand All @@ -129,3 +134,10 @@ func (r Controller) getDependantIngressFn(kind string) func(a client.Object) []r
return reqs
}
}

func (r Controller) isIngressNotFound(err error) bool {
if status := apierrors.APIStatus(nil); errors.As(err, &status) {
return status.Status().Reason == metav1.StatusReasonNotFound && status.Status().Kind == r.ingressKind
}
return false
}
69 changes: 23 additions & 46 deletions controllers/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,47 +5,43 @@ import (
"errors"
"fmt"

certmanagerv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/pomerium/ingress-controller/model"
)

func fetchIngress(
func (r *Controller) fetchIngress(
ctx context.Context,
c client.Client,
namespacedName types.NamespacedName,
) (*model.IngressConfig, error) {
ing := new(networkingv1.Ingress)
if err := c.Get(ctx, namespacedName, ing); err != nil {
ingress := new(networkingv1.Ingress)
if err := r.Client.Get(ctx, namespacedName, ingress); err != nil {
return nil, fmt.Errorf("get %s: %w", namespacedName.String(), err)
}

secrets, certs, err := fetchIngressSecrets(ctx, c, namespacedName.Namespace, ing)
secrets, err := r.fetchIngressSecrets(ctx, namespacedName.Namespace, ingress)
if err != nil {
// do not expose not found error
return nil, fmt.Errorf("tls: %s", err.Error())
return nil, fmt.Errorf("tls: %w", err)
}

svc, err := fetchIngressServices(ctx, c, namespacedName.Namespace, ing)
svc, err := r.fetchIngressServices(ctx, namespacedName.Namespace, ingress)
if err != nil {
return nil, fmt.Errorf("services: %s", err.Error())
return nil, fmt.Errorf("services: %w", err)
}
return &model.IngressConfig{
Ingress: ing,
Ingress: ingress,
Secrets: secrets,
Services: svc,
Certs: certs,
}, nil
}

// fetchIngressServices returns list of services referred from named port in the ingress path backend spec
func fetchIngressServices(ctx context.Context, c client.Client, namespace string, ing *networkingv1.Ingress) (map[types.NamespacedName]*corev1.Service, error) {
func (r *Controller) fetchIngressServices(ctx context.Context, namespace string, ingress *networkingv1.Ingress) (map[types.NamespacedName]*corev1.Service, error) {
sm := make(map[types.NamespacedName]*corev1.Service)
for _, rule := range ing.Spec.Rules {
for _, rule := range ingress.Spec.Rules {
if rule.HTTP == nil {
continue
}
Expand All @@ -56,7 +52,11 @@ func fetchIngressServices(ctx context.Context, c client.Client, namespace string
}
service := new(corev1.Service)
name := types.NamespacedName{Name: svc.Name, Namespace: namespace}
if err := c.Get(ctx, name, service); err != nil {
if err := r.Client.Get(ctx, name, service); err != nil {
if apierrors.IsNotFound(err) {
r.Registry.Add(model.ObjectKey(ingress), model.Key{Kind: r.serviceKind, NamespacedName: name})
}

return nil, fmt.Errorf("rule host=%s path=%s refers to service %s.%s port %s, failed to get service information: %w",
rule.Host, p.Path, namespace, svc.Name, svc.Port.Name, err)
}
Expand All @@ -66,48 +66,25 @@ func fetchIngressServices(ctx context.Context, c client.Client, namespace string
return sm, nil
}

func fetchIngressSecrets(ctx context.Context, c client.Client, namespace string, ingress *networkingv1.Ingress) (
func (r *Controller) fetchIngressSecrets(ctx context.Context, namespace string, ingress *networkingv1.Ingress) (
map[types.NamespacedName]*corev1.Secret,
map[types.NamespacedName]*certmanagerv1.Certificate,
error,
) {
isCM := isCertManager(ingress)
secrets := make(map[types.NamespacedName]*corev1.Secret)
certs := make(map[types.NamespacedName]*certmanagerv1.Certificate)
for _, tls := range ingress.Spec.TLS {
if tls.SecretName == "" {
return nil, nil, errors.New("tls.secretName is mandatory")
return nil, errors.New("tls.secretName is mandatory")
}
secret := new(corev1.Secret)
name := types.NamespacedName{Namespace: namespace, Name: tls.SecretName}

if isCM {
// cert-manager treats Ingress.tls.secretName as Certificate, not a secret name
// thus we need fetch Certificate first to learn an actual name of a secret
cert := new(certmanagerv1.Certificate)
if err := c.Get(ctx, name, cert); err != nil {
return nil, nil, fmt.Errorf("this ingress certs are managed by cert-manager, fetching Certificate designated by tls.secretName %s: %w", name.String(), err)
if err := r.Client.Get(ctx, name, secret); err != nil {
if apierrors.IsNotFound(err) {
r.Registry.Add(model.ObjectKey(ingress), model.Key{Kind: r.secretKind, NamespacedName: name})
}
certs[name] = cert
name.Name = cert.Spec.SecretName
}

if err := c.Get(ctx, name, secret); err != nil {
return nil, nil, fmt.Errorf("get secret %s: %w", name.String(), err)
return nil, fmt.Errorf("get secret %s: %w", name.String(), err)
}
secrets[name] = secret
}
return secrets, certs, nil
}

func isCertManager(ingress *networkingv1.Ingress) bool {
for _, a := range []string{
"cert-manager.io/issuer",
"cert-manager.io/cluster-issuer",
} {
if _, there := ingress.Annotations[a]; there {
return true
}
}
return false
return secrets, nil
}
31 changes: 22 additions & 9 deletions model/ingress_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,27 @@ func (ic *IngressConfig) GetServicePortByName(name types.NamespacedName, port st
return 0, fmt.Errorf("could not find port %s on service %s", port, name.String())
}

/*
func parseTLSSecret(secret *corev1.Secret) (*TLSSecret, error) {
if secret.Type != corev1.SecretTypeTLS {
return nil, fmt.Errorf("expected type %s, got %s", corev1.SecretTypeTLS, secret.Type)
type TLSCert struct {
Key []byte
Cert []byte
}

func (ic *IngressConfig) ParseTLSCerts() ([]*TLSCert, error) {
certs := make([]*TLSCert, 0, len(ic.Ingress.Spec.TLS))

for _, tls := range ic.Ingress.Spec.TLS {
secret := ic.Secrets[types.NamespacedName{Namespace: ic.Ingress.Namespace, Name: tls.SecretName}]
if secret == nil {
return nil, fmt.Errorf("tls.secretName=%s, but the secret wasn't fetched. this is a bug", tls.SecretName)
}
if secret.Type != corev1.SecretTypeTLS {
return nil, fmt.Errorf("tls.secretName=%s, expected type %s, got %s", tls.SecretName, corev1.SecretTypeTLS, secret.Type)
}
certs = append(certs, &TLSCert{
Key: secret.Data[corev1.TLSPrivateKeyKey],
Cert: secret.Data[corev1.TLSCertKey],
})
}
return &TLSSecret{
Key: secret.Data[corev1.TLSPrivateKeyKey],
Cert: secret.Data[corev1.TLSCertKey],
}, nil

return certs, nil
}
*/
102 changes: 102 additions & 0 deletions pomerium/cert_map.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package pomerium

import (
"crypto/x509"
"encoding/pem"
"fmt"
"strings"

pb "github.com/pomerium/pomerium/pkg/grpc/config"
)

type domainKey struct {
Host, Domain string
}

func parseDomainKey(dnsName string) domainKey {
parts := strings.SplitN(dnsName, ".", 2)
if len(parts) != 2 {
return domainKey{Host: dnsName}
}
return domainKey{Host: parts[0], Domain: parts[1]}
}

type certRef struct {
inUse bool
data *pb.Settings_Certificate
cert *x509.Certificate
}

func parseCert(cert *pb.Settings_Certificate) (*x509.Certificate, error) {
block, _ := pem.Decode(cert.CertBytes)
if block == nil {
return nil, fmt.Errorf("failed to decode cert block")
}

if block.Type != "CERTIFICATE" {
return nil, fmt.Errorf("expected CERTIFICATE PEM block, got %q", block.Type)
}

return x509.ParseCertificate(block.Bytes)
}

type domainMap map[domainKey]*certRef

func toDomainMap(certs []*pb.Settings_Certificate) (domainMap, error) {
domains := make(domainMap)
for _, cert := range certs {
crt, err := parseCert(cert)
if err != nil {
return nil, err
}
domains.add(cert, crt)
}
return domains, nil
}

func (dm domainMap) getCertsInUse() []*pb.Settings_Certificate {
certMap := make(map[*pb.Settings_Certificate]struct{})
for _, ref := range dm {
if ref.inUse {
certMap[ref.data] = struct{}{}
}
}
certs := make([]*pb.Settings_Certificate, 0, len(certMap))
for crt := range certMap {
certs = append(certs, crt)
}
return certs
}

func (dm domainMap) addIfNewer(key domainKey, ref *certRef) {
cur := dm[key]
if cur == nil {
dm[key] = ref
return
}
if cur.cert.NotAfter.Before(ref.cert.NotAfter) {
dm[key] = ref
}
}

func (dm domainMap) add(data *pb.Settings_Certificate, cert *x509.Certificate) {
ref := &certRef{
inUse: false,
data: data,
cert: cert,
}
for _, name := range cert.DNSNames {
dm.addIfNewer(parseDomainKey(name), ref)
}
}

func (dm domainMap) markInUse(dnsName string) {
key := parseDomainKey(dnsName)
if ref := dm[key]; ref != nil {
ref.inUse = true
return
}
if ref := dm[domainKey{Host: "*", Domain: key.Domain}]; ref != nil {
ref.inUse = true
}
}
Loading

0 comments on commit e9c2302

Please sign in to comment.