Skip to content

Commit e0ba40b

Browse files
author
Kubernetes Submit Queue
authored
Merge pull request kubernetes#39716 from zhouhaibing089/etcd-health-check
Automatic merge from submit-queue etcd component status check should include credentials - [x] Add TLS credentials into `pkg/genericapiserver.Backend`. - [x] Add TLS credentials into `pkg/registry/core/componentstatus.Server`. - [x] `pkg/probe/http.httpProber` should accept the TLS credentials. Now it is working. ```console $ kubectl get cs NAME STATUS MESSAGE ERROR scheduler Healthy ok controller-manager Healthy ok etcd-0 Healthy {"health": "true"} ``` Fixes kubernetes#27343.
2 parents a121d1c + b104017 commit e0ba40b

File tree

9 files changed

+93
-33
lines changed

9 files changed

+93
-33
lines changed

‎pkg/probe/http/http.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,12 @@ import (
3232

3333
func New() HTTPProber {
3434
tlsConfig := &tls.Config{InsecureSkipVerify: true}
35-
transport := utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: tlsConfig, DisableKeepAlives: true})
35+
return NewWithTLSConfig(tlsConfig)
36+
}
37+
38+
// NewWithTLSConfig takes tls config as parameter.
39+
func NewWithTLSConfig(config *tls.Config) HTTPProber {
40+
transport := utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: config, DisableKeepAlives: true})
3641
return httpProber{transport}
3742
}
3843

‎pkg/registry/core/componentstatus/rest.go

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,16 @@ import (
2727
"k8s.io/apiserver/pkg/registry/rest"
2828
"k8s.io/kubernetes/pkg/api"
2929
"k8s.io/kubernetes/pkg/probe"
30-
httpprober "k8s.io/kubernetes/pkg/probe/http"
3130
)
3231

3332
type REST struct {
34-
GetServersToValidate func() map[string]Server
35-
prober httpprober.HTTPProber
33+
GetServersToValidate func() map[string]*Server
3634
}
3735

3836
// NewStorage returns a new REST.
39-
func NewStorage(serverRetriever func() map[string]Server) *REST {
37+
func NewStorage(serverRetriever func() map[string]*Server) *REST {
4038
return &REST{
4139
GetServersToValidate: serverRetriever,
42-
prober: httpprober.New(),
4340
}
4441
}
4542

@@ -60,7 +57,7 @@ func (rs *REST) List(ctx genericapirequest.Context, options *metainternalversion
6057
wait.Add(len(servers))
6158
statuses := make(chan api.ComponentStatus, len(servers))
6259
for k, v := range servers {
63-
go func(name string, server Server) {
60+
go func(name string, server *Server) {
6461
defer wait.Done()
6562
status := rs.getComponentStatus(name, server)
6663
statuses <- *status
@@ -97,8 +94,8 @@ func ToConditionStatus(s probe.Result) api.ConditionStatus {
9794
}
9895
}
9996

100-
func (rs *REST) getComponentStatus(name string, server Server) *api.ComponentStatus {
101-
status, msg, err := server.DoServerCheck(rs.prober)
97+
func (rs *REST) getComponentStatus(name string, server *Server) *api.ComponentStatus {
98+
status, msg, err := server.DoServerCheck()
10299
errorMsg := ""
103100
if err != nil {
104101
errorMsg = err.Error()

‎pkg/registry/core/componentstatus/rest_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,17 @@ type testResponse struct {
5050
}
5151

5252
func NewTestREST(resp testResponse) *REST {
53+
prober := &fakeHttpProber{
54+
result: resp.result,
55+
body: resp.data,
56+
err: resp.err,
57+
}
5358
return &REST{
54-
GetServersToValidate: func() map[string]Server {
55-
return map[string]Server{
56-
"test1": {Addr: "testserver1", Port: 8000, Path: "/healthz"},
59+
GetServersToValidate: func() map[string]*Server {
60+
return map[string]*Server{
61+
"test1": {Addr: "testserver1", Port: 8000, Path: "/healthz", Prober: prober},
5762
}
5863
},
59-
prober: &fakeHttpProber{
60-
result: resp.result,
61-
body: resp.data,
62-
err: resp.err,
63-
},
6464
}
6565
}
6666

‎pkg/registry/core/componentstatus/validator.go

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ limitations under the License.
1717
package componentstatus
1818

1919
import (
20+
"crypto/tls"
2021
"net/http"
21-
22+
"sync"
2223
"time"
2324

2425
utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -42,7 +43,10 @@ type Server struct {
4243
Port int
4344
Path string
4445
EnableHTTPS bool
46+
TLSConfig *tls.Config
4547
Validate ValidatorFn
48+
Prober httpprober.HTTPProber
49+
Once sync.Once
4650
}
4751

4852
type ServerStatus struct {
@@ -58,14 +62,22 @@ type ServerStatus struct {
5862
Err string `json:"err,omitempty"`
5963
}
6064

61-
func (server *Server) DoServerCheck(prober httpprober.HTTPProber) (probe.Result, string, error) {
65+
func (server *Server) DoServerCheck() (probe.Result, string, error) {
66+
// setup the prober
67+
server.Once.Do(func() {
68+
if server.Prober != nil {
69+
return
70+
}
71+
server.Prober = httpprober.NewWithTLSConfig(server.TLSConfig)
72+
})
73+
6274
scheme := "http"
6375
if server.EnableHTTPS {
6476
scheme = "https"
6577
}
6678
url := utilnet.FormatURL(scheme, server.Addr, server.Port, server.Path)
6779

68-
result, data, err := prober.Probe(url, nil, probeTimeOut)
80+
result, data, err := server.Prober.Probe(url, nil, probeTimeOut)
6981

7082
if err != nil {
7183
return probe.Unknown, "", err

‎pkg/registry/core/componentstatus/validator_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ func TestValidate(t *testing.T) {
5959
}
6060

6161
s.Validate = test.validator
62-
result, data, err := s.DoServerCheck(fakeProber)
62+
s.Prober = fakeProber
63+
result, data, err := s.DoServerCheck()
6364
if test.expectErr && err == nil {
6465
t.Error("unexpected non-error")
6566
}

‎pkg/registry/core/rest/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ go_test(
1515
tags = ["automanaged"],
1616
deps = [
1717
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
18+
"//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
1819
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
1920
],
2021
)

‎pkg/registry/core/rest/storage_core.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -242,14 +242,14 @@ type componentStatusStorage struct {
242242
storageFactory serverstorage.StorageFactory
243243
}
244244

245-
func (s componentStatusStorage) serversToValidate() map[string]componentstatus.Server {
246-
serversToValidate := map[string]componentstatus.Server{
245+
func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server {
246+
serversToValidate := map[string]*componentstatus.Server{
247247
"controller-manager": {Addr: "127.0.0.1", Port: ports.ControllerManagerPort, Path: "/healthz"},
248248
"scheduler": {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"},
249249
}
250250

251251
for ix, machine := range s.storageFactory.Backends() {
252-
etcdUrl, err := url.Parse(machine)
252+
etcdUrl, err := url.Parse(machine.Server)
253253
if err != nil {
254254
glog.Errorf("Failed to parse etcd url for validation: %v", err)
255255
continue
@@ -269,9 +269,10 @@ func (s componentStatusStorage) serversToValidate() map[string]componentstatus.S
269269
port = 2379
270270
}
271271
// TODO: etcd health checking should be abstracted in the storage tier
272-
serversToValidate[fmt.Sprintf("etcd-%d", ix)] = componentstatus.Server{
272+
serversToValidate[fmt.Sprintf("etcd-%d", ix)] = &componentstatus.Server{
273273
Addr: addr,
274274
EnableHTTPS: etcdUrl.Scheme == "https",
275+
TLSConfig: machine.TLSConfig,
275276
Port: port,
276277
Path: "/health",
277278
Validate: etcdutil.EtcdHealthCheck,

‎pkg/registry/core/rest/storage_core_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"testing"
2121

2222
"k8s.io/apimachinery/pkg/runtime/schema"
23+
"k8s.io/apiserver/pkg/server/storage"
2324
"k8s.io/apiserver/pkg/storage/storagebackend"
2425
)
2526

@@ -47,6 +48,6 @@ func (f fakeStorageFactory) ResourcePrefix(groupResource schema.GroupResource) s
4748
return ""
4849
}
4950

50-
func (f fakeStorageFactory) Backends() []string {
51-
return []string{"etcd-0"}
51+
func (f fakeStorageFactory) Backends() []storage.Backend {
52+
return []storage.Backend{{Server: "etcd-0"}}
5253
}

‎staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ limitations under the License.
1717
package storage
1818

1919
import (
20+
"crypto/tls"
21+
"crypto/x509"
22+
"io/ioutil"
2023
"strings"
2124

2225
"github.com/golang/glog"
@@ -27,6 +30,15 @@ import (
2730
"k8s.io/apiserver/pkg/storage/storagebackend"
2831
)
2932

33+
// Backend describes the storage servers, the information here should be enough
34+
// for health validations.
35+
type Backend struct {
36+
// the url of storage backend like: https://etcd.domain:2379
37+
Server string
38+
// the required tls config
39+
TLSConfig *tls.Config
40+
}
41+
3042
// StorageFactory is the interface to locate the storage for a given GroupResource
3143
type StorageFactory interface {
3244
// New finds the storage destination for the given group and resource. It will
@@ -40,7 +52,7 @@ type StorageFactory interface {
4052

4153
// Backends gets all backends for all registered storage destinations.
4254
// Used for getting all instances for health validations.
43-
Backends() []string
55+
Backends() []Backend
4456
}
4557

4658
// DefaultStorageFactory takes a GroupResource and returns back its storage interface. This result includes:
@@ -252,15 +264,45 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
252264
return &storageConfig, nil
253265
}
254266

255-
// Get all backends for all registered storage destinations.
267+
// Backends returns all backends for all registered storage destinations.
256268
// Used for getting all instances for health validations.
257-
func (s *DefaultStorageFactory) Backends() []string {
258-
backends := sets.NewString(s.StorageConfig.ServerList...)
269+
func (s *DefaultStorageFactory) Backends() []Backend {
270+
servers := sets.NewString(s.StorageConfig.ServerList...)
259271

260272
for _, overrides := range s.Overrides {
261-
backends.Insert(overrides.etcdLocation...)
273+
servers.Insert(overrides.etcdLocation...)
274+
}
275+
276+
tlsConfig := &tls.Config{
277+
InsecureSkipVerify: true,
278+
}
279+
if len(s.StorageConfig.CertFile) > 0 && len(s.StorageConfig.KeyFile) > 0 {
280+
cert, err := tls.LoadX509KeyPair(s.StorageConfig.CertFile, s.StorageConfig.KeyFile)
281+
if err != nil {
282+
glog.Errorf("failed to load key pair while getting backends: %s", err)
283+
} else {
284+
tlsConfig.Certificates = []tls.Certificate{cert}
285+
}
286+
}
287+
if len(s.StorageConfig.CAFile) > 0 {
288+
if caCert, err := ioutil.ReadFile(s.StorageConfig.CAFile); err != nil {
289+
glog.Errorf("failed to read ca file while getting backends: %s", err)
290+
} else {
291+
caPool := x509.NewCertPool()
292+
caPool.AppendCertsFromPEM(caCert)
293+
tlsConfig.RootCAs = caPool
294+
tlsConfig.InsecureSkipVerify = false
295+
}
296+
}
297+
298+
backends := []Backend{}
299+
for server := range servers {
300+
backends = append(backends, Backend{
301+
Server: server,
302+
TLSConfig: tlsConfig,
303+
})
262304
}
263-
return backends.List()
305+
return backends
264306
}
265307

266308
func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string {

0 commit comments

Comments
 (0)