grpc: Enable client-side health_v1 health checking

Author: beautifulentropy

grpc/client.go

@@ -14,11 +14,13 @@ import (
 	"github.com/letsencrypt/boulder/cmd"
 	bcreds "github.com/letsencrypt/boulder/grpc/creds"
 
-	// 'grpc/health' is imported for its init function, which causes clients to
-	// rely on the Health Service for load-balancing.
 	// 'grpc/internal/resolver/dns' is imported for its init function, which
 	// registers the SRV resolver.
 	"google.golang.org/grpc/balancer/roundrobin"
+
+	// 'grpc/health' is imported for its init function, which causes clients to
+	// rely on the Health Service for load-balancing as long as a
+	// "healthCheckConfig" is specified in the gRPC service config.
 	_ "google.golang.org/grpc/health"
 
 	_ "github.com/letsencrypt/boulder/grpc/internal/resolver/dns"
@@ -61,7 +63,21 @@ func ClientSetup(c *cmd.GRPCClientConfig, tlsConfig *tls.Config, statsRegistry p
 	creds := bcreds.NewClientCredentials(tlsConfig.RootCAs, tlsConfig.Certificates, hostOverride)
 	return grpc.NewClient(
 		target,
-		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, roundrobin.Name)),
+		grpc.WithDefaultServiceConfig(
+			fmt.Sprintf(
+				// By setting the service name to an empty string in
+				// healthCheckConfig, we're instructing the gRPC client to query
+				// the overall health status of each server. The grpc-go health
+				// server, as constructed by health.NewServer(), unconditionally
+				// sets the overall service (e.g. "") status to SERVING. If a
+				// specific service name were set, the server would need to
+				// explicitly transition that service to SERVING; otherwise,
+				// clients would receive a NOT_FOUND status and the connection
+				// would be marked as unhealthy (TRANSIENT_FAILURE).
+				`{"healthCheckConfig": {"serviceName": ""},"loadBalancingConfig": [{"%s":{}}]}`,
+				roundrobin.Name,
+			),
+		),
 		grpc.WithTransportCredentials(creds),
 		grpc.WithChainUnaryInterceptor(unaryInterceptors...),
 		grpc.WithChainStreamInterceptor(streamInterceptors...),

grpc/server.go

@@ -129,6 +129,15 @@ func (sb *serverBuilder) Build(tlsConfig *tls.Config, statsRegistry prometheus.R
 		}
 	}
 
+	// Ensure that the health service has the same ClientNames as the other
+	// services, so that health checks can be performed by clients which are
+	// allowed to connect to the server.
+	healthService := sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName]
+	for as := range acceptedSANs {
+		healthService.ClientNames = append(healthService.ClientNames, as)
+	}
+	sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName] = healthService
+
 	creds, err := bcreds.NewServerCredentials(tlsConfig, acceptedSANs)
 	if err != nil {
 		return nil, err
@@ -224,6 +233,9 @@ func (sb *serverBuilder) Build(tlsConfig *tls.Config, statsRegistry prometheus.R
 
 // initLongRunningCheck initializes a goroutine which will periodically check
 // the health of the provided service and update the health server accordingly.
+//
+// TODO(): Remove the service parameter and instead rely on transitioning the
+// overall health of the server (e.g. "") instead of individual services.
 func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, service string, checkImpl func(context.Context) error) {
 	// Set the initial health status for the service.
 	sb.healthSrv.SetServingStatus(service, healthpb.HealthCheckResponse_NOT_SERVING)
@@ -253,6 +265,7 @@ func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, servi
 		} else {
 			sb.logger.Infof("transitioning health of %q from %q to %q", service, last, next)
 		}
+		sb.healthSrv.SetServingStatus("", next)
 		sb.healthSrv.SetServingStatus(service, next)
 		return next
 	}

test/consul/config.hcl

@@ -240,24 +240,6 @@ services {
       tls_server_name = "sa.boulder"
       tls_skip_verify = false
       interval        = "2s"
-    },
-    {
-      id              = "sa-a-grpc-sa"
-      name            = "sa-a-grpc-sa"
-      grpc            = "10.77.77.77:9395/sa.StorageAuthority"
-      grpc_use_tls    = true
-      tls_server_name = "sa.boulder"
-      tls_skip_verify = false
-      interval        = "2s"
-    },
-    {
-      id              = "sa-a-grpc-saro"
-      name            = "sa-a-grpc-saro"
-      grpc            = "10.77.77.77:9395/sa.StorageAuthorityReadOnly"
-      grpc_use_tls    = true
-      tls_server_name = "sa.boulder"
-      tls_skip_verify = false
-      interval        = "2s"
     }
   ]
 }