From c162e41585b4a66d2f24389aac24eadf9c371e75 Mon Sep 17 00:00:00 2001 From: Drumil Patel <42701709+weastel@users.noreply.github.com> Date: Sat, 5 Sep 2020 21:00:49 +0530 Subject: [PATCH] Add EKS provider for infra (#384) This change adds EKS commands for the creation/deletion of clusters, nodegroups, and resources similar to the existing GKE commands in infra. The documentation for deploying prombench for EKS is also added. The changes in this commit are not sufficient for running funcbench on EKS. Signed-off-by: Drumil Patel --- funcbench/Makefile | 4 + funcbench/manifests/cluster_eks.yaml | 26 + go.mod | 2 + go.sum | 54 ++ infra/README.md | 33 + infra/infra.go | 45 ++ pkg/provider/eks/eks.go | 605 +++++++++++++++++++ pkg/provider/k8s/k8s.go | 10 +- pkg/provider/provider.go | 12 +- prombench/Makefile | 56 +- prombench/README.md | 3 +- prombench/docs/eks.md | 113 ++++ prombench/manifests/cluster_eks.yaml | 26 + prombench/manifests/prombench/nodes_eks.yaml | 35 ++ 14 files changed, 997 insertions(+), 27 deletions(-) create mode 100644 funcbench/manifests/cluster_eks.yaml create mode 100644 pkg/provider/eks/eks.go create mode 100644 prombench/docs/eks.md create mode 100644 prombench/manifests/cluster_eks.yaml create mode 100644 prombench/manifests/prombench/nodes_eks.yaml diff --git a/funcbench/Makefile b/funcbench/Makefile index 7eae4c546..703ff1fd9 100644 --- a/funcbench/Makefile +++ b/funcbench/Makefile @@ -14,11 +14,15 @@ clean: resource_delete cluster_delete cluster_create: $(INFRA_CMD) $(PROVIDER) cluster create -a ${AUTH_FILE} \ -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \ -f manifests/cluster_$(PROVIDER).yaml cluster_delete: $(INFRA_CMD) $(PROVIDER) cluster delete -a ${AUTH_FILE} \ -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \ -f manifests/cluster_$(PROVIDER).yaml resource_apply: diff --git a/funcbench/manifests/cluster_eks.yaml b/funcbench/manifests/cluster_eks.yaml new file mode 100644 index 000000000..350ec3b7b --- /dev/null +++ b/funcbench/manifests/cluster_eks.yaml @@ -0,0 +1,26 @@ +cluster: + name: {{ .CLUSTER_NAME }} + version: 1.16 + rolearn: {{ .ROLE_ARN }} + resourcesvpcconfig: + endpointpublicaccess: true + subnetids: + {{ range $subnetId := split .SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} +nodegroups: + - nodegroupname: {{ .CLUSTER_NAME }} + noderole: {{ .NODE_ROLE }} + disksize: 100 + subnets: + {{ range $subnetId := split .SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} + instancetypes: + - r6g.xlarge + scalingconfig: + desiredsize: 1 + maxsize: 1 + minsize: 1 + labels: + node-name: funcbench-{{ .PR_NUMBER }} diff --git a/go.mod b/go.mod index 12d447486..49d36a3e2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( cloud.google.com/go v0.56.0 + github.com/aws/aws-sdk-go v1.34.5 github.com/go-git/go-git-fixtures/v4 v4.0.1 github.com/go-git/go-git/v5 v5.1.0 github.com/google/go-github/v29 v29.0.3 @@ -24,5 +25,6 @@ require ( k8s.io/apiextensions-apiserver v0.18.4 k8s.io/apimachinery v0.18.4 k8s.io/client-go v0.18.4 + sigs.k8s.io/aws-iam-authenticator v0.5.1 sigs.k8s.io/kind v0.8.1 ) diff --git a/go.sum b/go.sum index 75952b6a4..2e5ea6d9f 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,9 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.5 h1:FwubVVX9u+kW9qDCjVzyWOdsL+W5wPq683wMk2R2GXk= +github.com/aws/aws-sdk-go v1.34.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -124,6 +127,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -139,6 +143,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -249,6 +254,7 @@ github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -275,10 +281,13 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g= +github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -326,6 +335,7 @@ github.com/google/go-github/v29 v29.0.3 h1:IktKCTwU//aFHnpA+2SLIi7Oo9uhAzgsdZNbc github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -409,8 +419,11 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= @@ -479,6 +492,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -508,9 +522,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -539,6 +555,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -546,10 +563,12 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/alertmanager v0.21.0 h1:qK51JcUR9l/unhawGA9F9B64OCYfcGewhPNprem/Acc= github.com/prometheus/alertmanager v0.21.0/go.mod h1:h7tJ81NA0VLWvWEayi1QltevFkLF3KxmC/malTcT8Go= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= @@ -561,24 +580,29 @@ github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -632,6 +656,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -658,6 +683,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.hein.dev/go-version v0.1.0/go.mod h1:WOEm7DWMroRe5GdUgHMvx+Pji5WWIpMuXmK/3foylXs= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -699,7 +725,9 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -809,6 +837,9 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812172437-4e8604ab3aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -850,6 +881,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -867,6 +899,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190812233024-afc3694995b6/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -898,6 +931,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.0.0-20170206182103-3d017632ea10/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1001,34 +1037,52 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.16.8/go.mod h1:a8EOdYHO8en+YHhPBLiW5q+3RfHTr7wxTqqp7emJ7PM= k8s.io/api v0.18.4 h1:8x49nBRxuXGUlDlwlWd3RMY1SayZrzFfxea3UZSkFw4= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= k8s.io/apiextensions-apiserver v0.18.4 h1:Y3HGERmS8t9u12YNUFoOISqefaoGRuTc43AYCLzWmWE= k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apimachinery v0.16.8/go.mod h1:Xk2vD2TRRpuWYLQNM6lT9R7DSFZUYG03SarNkbGrnKE= k8s.io/apimachinery v0.18.4 h1:ST2beySjhqwJoIFk6p7Hp5v5O0hYY6Gngq/gUYXTPIA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= +k8s.io/client-go v0.16.8/go.mod h1:WmPuN0yJTKHXoklExKxzo3jSXmr3EnN+65uaTb5VuNs= k8s.io/client-go v0.18.4 h1:un55V1Q/B3JO3A76eS0kUSywgGK/WR3BQ8fHQjNa6Zc= k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g= +k8s.io/code-generator v0.16.8/go.mod h1:wFdrXdVi/UC+xIfLi+4l9elsTT/uEF61IfcN2wOLULQ= k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.16.8/go.mod h1:Q8UWOWShpP3MZZny4n/15gOncfaaVtc9SbCdkM5MhUE= k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/sample-controller v0.16.8/go.mod h1:aXlORS1ekU77qhGybB5t3JORDurzDpWgvMYxmCsiuos= +k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/kind v0.8.1 h1:9wsEbEtMQV9QObaqS/T4VxBeXXPtu+qM9sFMqgO/90o= sigs.k8s.io/kind v0.8.1/go.mod h1:oNKTxUVPYkV9lWzY6CVMNluVq8cBsyq+UgPJdvA3uu4= +sigs.k8s.io/aws-iam-authenticator v0.5.1 h1:0Nv09uOayy99IOYgNamMl0cwTuQWRtEuUu6s3mSgyEs= +sigs.k8s.io/aws-iam-authenticator v0.5.1/go.mod h1:yPDLi58MDx1UtCrRMOykLm1IyKKPGHgcGCafcbn2s3E= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/infra/README.md b/infra/README.md index c3e1f4578..efd4b4e49 100644 --- a/infra/README.md +++ b/infra/README.md @@ -79,6 +79,39 @@ Commands: kind resource delete -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2 + eks info + eks info -v hashStable:COMMIT1 -v hashTesting:COMMIT2 + + eks cluster create + eks cluster create -a credentials -f FileOrFolder + + eks cluster delete + eks cluster delete -a credentials -f FileOrFolder + + eks nodes create + eks nodes create -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v + CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3 + + eks nodes delete + eks nodes delete -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v + CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3 + + eks nodes check-running + eks nodes check-running -a credentails -f FileOrFolder -v ZONE:eu-west-1 -v + CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3 + + eks nodes check-deleted + eks nodes check-deleted -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v + CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3 + + eks resource apply + eks resource apply -a credentials -f manifestsFileOrFolder -v + hashStable:COMMIT1 -v hashTesting:COMMIT2 + + eks resource delete + eks resource delete -a credentials -f manifestsFileOrFolder -v + hashStable:COMMIT1 -v hashTesting:COMMIT2 + ``` diff --git a/infra/infra.go b/infra/infra.go index ecc2da8b2..aa0859fbb 100644 --- a/infra/infra.go +++ b/infra/infra.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/test-infra/pkg/provider" + "github.com/prometheus/test-infra/pkg/provider/eks" "github.com/prometheus/test-infra/pkg/provider/gke" kind "github.com/prometheus/test-infra/pkg/provider/kind" "gopkg.in/alecthomas/kingpin.v2" @@ -107,6 +108,50 @@ func main() { k8sKINDResource.Command("delete", "kind resource delete -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2"). Action(k.ResourceDelete) + // EKS based commands + e := eks.New(dr) + k8sEKS := app.Command("eks", "Amazon Elastic Kubernetes Service - https://aws.amazon.com/eks"). + Action(e.SetupDeploymentResources) + k8sEKS.Flag("auth", "filename which consist eks credentials."). + PlaceHolder("credentials"). + Short('a'). + StringVar(&e.Auth) + + k8sEKS.Command("info", "eks info -v hashStable:COMMIT1 -v hashTesting:COMMIT2"). + Action(e.GetDeploymentVars) + + // EKS Cluster operations + k8sEKSCluster := k8sEKS.Command("cluster", "manage EKS clusters"). + Action(e.NewEKSClient). + Action(e.EKSDeploymentParse) + k8sEKSCluster.Command("create", "eks cluster create -a credentials -f FileOrFolder"). + Action(e.ClusterCreate) + k8sEKSCluster.Command("delete", "eks cluster delete -a credentials -f FileOrFolder"). + Action(e.ClusterDelete) + + // Cluster node-pool operations + k8sEKSNodeGroup := k8sEKS.Command("nodes", "manage EKS clusters nodegroups"). + Action(e.NewEKSClient). + Action(e.EKSDeploymentParse) + k8sEKSNodeGroup.Command("create", "eks nodes create -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3"). + Action(e.NodeGroupCreate) + k8sEKSNodeGroup.Command("delete", "eks nodes delete -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3"). + Action(e.NodeGroupDelete) + k8sEKSNodeGroup.Command("check-running", "eks nodes check-running -a credentails -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3"). + Action(e.AllNodeGroupsRunning) + k8sEKSNodeGroup.Command("check-deleted", "eks nodes check-deleted -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3"). + Action(e.AllNodeGroupsDeleted) + + // K8s resource operations. + k8sEKSResource := k8sEKS.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.Required variables -v ZONE:us-east-2 -v CLUSTER_NAME:test `). + Action(e.NewEKSClient). + Action(e.K8SDeploymentsParse). + Action(e.NewK8sProvider) + k8sEKSResource.Command("apply", "eks resource apply -a credentials -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2"). + Action(e.ResourceApply) + k8sEKSResource.Command("delete", "eks resource delete -a credentials -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2"). + Action(e.ResourceDelete) + if _, err := app.Parse(os.Args[1:]); err != nil { fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments")) app.Usage(os.Args[1:]) diff --git a/pkg/provider/eks/eks.go b/pkg/provider/eks/eks.go new file mode 100644 index 000000000..079b16871 --- /dev/null +++ b/pkg/provider/eks/eks.go @@ -0,0 +1,605 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eks + +import ( + "context" + "encoding/base64" + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + awsSession "github.com/aws/aws-sdk-go/aws/session" + eks "github.com/aws/aws-sdk-go/service/eks" + "github.com/pkg/errors" + k8sProvider "github.com/prometheus/test-infra/pkg/provider/k8s" + + "github.com/prometheus/test-infra/pkg/provider" + "gopkg.in/alecthomas/kingpin.v2" + yamlGo "gopkg.in/yaml.v2" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + awsToken "sigs.k8s.io/aws-iam-authenticator/pkg/token" +) + +type Resource = provider.Resource + +type eksCluster struct { + Cluster eks.CreateClusterInput + NodeGroups []eks.CreateNodegroupInput +} + +// EKS holds the fields used to generate an API request. +type EKS struct { + Auth string + + ClusterName string + // The eks client used when performing EKS requests. + clientEKS *eks.EKS + // The aws session used in abstraction of aws credentials. + sessionAWS *awsSession.Session + // The k8s provider used when we work with the manifest files. + k8sProvider *k8sProvider.K8s + // Final DeploymentFiles files. + DeploymentFiles []string + // Final DeploymentVars. + DeploymentVars map[string]string + // DeployResource to construct DeploymentVars and DeploymentFiles + DeploymentResource *provider.DeploymentResource + // Content bytes after parsing the template variables, grouped by filename. + eksResources []Resource + // K8s resource.runtime objects after parsing the template variables, grouped by filename. + k8sResources []k8sProvider.Resource + + ctx context.Context +} + +// New is the EKS constructor +func New(dr *provider.DeploymentResource) *EKS { + eks := &EKS{ + DeploymentResource: dr, + } + return eks +} + +// NewEKSClient sets the EKS client used when performing the GKE requests. +func (c *EKS) NewEKSClient(*kingpin.ParseContext) error { + if c.Auth != "" { + } else if c.Auth = os.Getenv("AWS_APPLICATION_CREDENTIALS"); c.Auth == "" { + return errors.Errorf("no auth provided set the auth flag or the AWS_APPLICATION_CREDENTIALS env variable") + } + + // When the auth variable points to a file + // put the file content in the variable. + if content, err := ioutil.ReadFile(c.Auth); err == nil { + c.Auth = string(content) + } + + // Check if auth data is base64 encoded and decode it. + encoded, err := regexp.MatchString("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$", c.Auth) + if err != nil { + return err + } + if encoded { + auth, err := base64.StdEncoding.DecodeString(c.Auth) + if err != nil { + return errors.Wrap(err, "could not decode auth data") + } + c.Auth = string(auth) + } + + credValue := &credentials.Value{} + if err = yamlGo.UnmarshalStrict([]byte(c.Auth), credValue); err != nil { + return errors.Wrap(err, "could not get credential values") + } + + awsSess := awsSession.Must(awsSession.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentialsFromCreds(*credValue), + Region: aws.String(c.DeploymentVars["ZONE"]), + })) + + c.sessionAWS = awsSess + c.clientEKS = eks.New(awsSess) + c.ctx = context.Background() + return nil +} + +// checkDeploymentVarsAndFiles checks whether the requied deployment vars are passed. +func (c *EKS) checkDeploymentVarsAndFiles() error { + reqDepVars := []string{"ZONE", "CLUSTER_NAME"} + for _, k := range reqDepVars { + if v := c.DeploymentVars[k]; v == "" { + return fmt.Errorf("missing required %v variable", k) + } + } + if len(c.DeploymentFiles) == 0 { + return fmt.Errorf("missing deployment file(s)") + } + return nil +} + +// SetupDeploymentResources Sets up DeploymentVars and DeploymentFiles +func (c *EKS) SetupDeploymentResources(*kingpin.ParseContext) error { + c.DeploymentFiles = c.DeploymentResource.DeploymentFiles + c.DeploymentVars = provider.MergeDeploymentVars( + c.DeploymentResource.DefaultDeploymentVars, + c.DeploymentResource.FlagDeploymentVars, + ) + return nil +} + +// EKSDeploymentParse parses the cluster/nodegroups deployment file and saves the result as bytes grouped by the filename. +// Any variables passed to the cli will be replaced in the resource files following the golang text template format. +func (c *EKS) EKSDeploymentParse(*kingpin.ParseContext) error { + if err := c.checkDeploymentVarsAndFiles(); err != nil { + return err + } + + deploymentResource, err := provider.DeploymentsParse(c.DeploymentFiles, c.DeploymentVars) + if err != nil { + return fmt.Errorf("Couldn't parse deployment files: %v", err) + } + + c.eksResources = deploymentResource + return nil +} + +// K8SDeploymentsParse parses the k8s objects deployment files and saves the result as k8s objects grouped by the filename. +// Any variables passed to the cli will be replaced in the resources files following the golang text template format. +func (c *EKS) K8SDeploymentsParse(*kingpin.ParseContext) error { + if err := c.checkDeploymentVarsAndFiles(); err != nil { + return err + } + + deploymentResource, err := provider.DeploymentsParse(c.DeploymentFiles, c.DeploymentVars) + if err != nil { + return fmt.Errorf("Couldn't parse deployment files: %v", err) + } + + for _, deployment := range deploymentResource { + + decode := scheme.Codecs.UniversalDeserializer().Decode + k8sObjects := make([]runtime.Object, 0) + + for _, text := range strings.Split(string(deployment.Content), provider.Separator) { + text = strings.TrimSpace(text) + if len(text) == 0 { + continue + } + + resource, _, err := decode([]byte(text), nil, nil) + + if err != nil { + return errors.Wrapf(err, "decoding the resource file:%v, section:%v...", deployment.FileName, text[:100]) + } + if resource == nil { + continue + } + k8sObjects = append(k8sObjects, resource) + } + if len(k8sObjects) > 0 { + c.k8sResources = append(c.k8sResources, k8sProvider.Resource{FileName: deployment.FileName, Objects: k8sObjects}) + } + } + return nil +} + +// ClusterCreate create a new cluster or applies changes to an existing cluster. +func (c *EKS) ClusterCreate(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + + log.Printf("Cluster create request: name:'%s'", *req.Cluster.Name) + _, err := c.clientEKS.CreateCluster(&req.Cluster) + if err != nil { + return fmt.Errorf("Couldn't create cluster '%v', file:%v ,err: %v", *req.Cluster.Name, deployment.FileName, err) + } + + err = provider.RetryUntilTrue( + fmt.Sprintf("creating cluster:%v", *req.Cluster.Name), + provider.EKSRetryCount, + func() (bool, error) { return c.clusterRunning(*req.Cluster.Name) }, + ) + + if err != nil { + return fmt.Errorf("creating cluster err:%v", err) + } + + for _, nodegroupReq := range req.NodeGroups { + nodegroupReq.ClusterName = req.Cluster.Name + log.Printf("Nodegroup create request: NodeGroupName: '%s', ClusterName: '%s'", *nodegroupReq.NodegroupName, *req.Cluster.Name) + _, err := c.clientEKS.CreateNodegroup(&nodegroupReq) + if err != nil { + return fmt.Errorf("Couldn't create nodegroup '%v' for cluster '%v, file:%v ,err: %v", nodegroupReq.NodegroupName, req.Cluster.Name, deployment.FileName, err) + } + + err = provider.RetryUntilTrue( + fmt.Sprintf("creating nodegroup:%s for cluster:%s", *nodegroupReq.NodegroupName, *req.Cluster.Name), + provider.EKSRetryCount, + func() (bool, error) { return c.nodeGroupCreated(*nodegroupReq.NodegroupName, *req.Cluster.Name) }, + ) + + if err != nil { + return fmt.Errorf("creating nodegroup err:%v", err) + } + } + } + return nil +} + +// ClusterDelete deletes a eks Cluster +func (c *EKS) ClusterDelete(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + + // To delete a cluster we have to manually delete all cluster + log.Printf("Removing all nodepools for '%s'", *req.Cluster.Name) + + // Listing all nodepools for cluster + reqL := &eks.ListNodegroupsInput{ + ClusterName: req.Cluster.Name, + } + + for { + resL, err := c.clientEKS.ListNodegroups(reqL) + if err != nil { + return fmt.Errorf("listing nodepools err:%v", err) + } + + for _, nodegroup := range resL.Nodegroups { + log.Printf("Removing nodepool '%s' in cluster '%s'", *nodegroup, *req.Cluster.Name) + + reqD := eks.DeleteNodegroupInput{ + ClusterName: req.Cluster.Name, + NodegroupName: nodegroup, + } + _, err := c.clientEKS.DeleteNodegroup(&reqD) + if err != nil { + return fmt.Errorf("Couldn't create nodegroup '%v' for cluster '%v ,err: %v", *nodegroup, req.Cluster.Name, err) + } + + err = provider.RetryUntilTrue( + fmt.Sprintf("deleting nodegroup:%v for cluster:%v", *nodegroup, *req.Cluster.Name), + provider.GlobalRetryCount, + func() (bool, error) { return c.nodeGroupDeleted(*nodegroup, *req.Cluster.Name) }, + ) + + if err != nil { + return fmt.Errorf("deleting nodegroup err:%v", err) + } + } + + if resL.NextToken == nil { + break + } else { + reqL.NextToken = resL.NextToken + } + } + + reqD := &eks.DeleteClusterInput{ + Name: req.Cluster.Name, + } + + log.Printf("Removing cluster '%v'", *reqD.Name) + _, err := c.clientEKS.DeleteCluster(reqD) + if err != nil { + return fmt.Errorf("Couldn't delete cluster '%v', file:%v ,err: %v", *req.Cluster.Name, deployment.FileName, err) + } + + err = provider.RetryUntilTrue( + fmt.Sprintf("deleting cluster:%v", *reqD.Name), + provider.GlobalRetryCount, + func() (bool, error) { return c.clusterDeleted(*reqD.Name) }) + + if err != nil { + return fmt.Errorf("removing cluster err:%v", err) + } + } + return nil +} + +// clusterRunning checks whether a cluster is in a active state. +func (c *EKS) clusterRunning(name string) (bool, error) { + req := &eks.DescribeClusterInput{ + Name: aws.String(name), + } + clusterRes, err := c.clientEKS.DescribeCluster(req) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == eks.ErrCodeNotFoundException { + return false, nil + } + return false, fmt.Errorf("Couldn't get cluster status: %v", err) + } + if *clusterRes.Cluster.Status == eks.ClusterStatusFailed { + return false, fmt.Errorf("Cluster not in a status to become ready - %s", *clusterRes.Cluster.Status) + } + if *clusterRes.Cluster.Status == eks.ClusterStatusActive { + return true, nil + } + log.Printf("Cluster '%v' status: %v", name, *clusterRes.Cluster.Status) + return false, nil +} + +func (c *EKS) clusterDeleted(name string) (bool, error) { + req := &eks.DescribeClusterInput{ + Name: aws.String(name), + } + clusterRes, err := c.clientEKS.DescribeCluster(req) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == eks.ErrCodeResourceNotFoundException { + return true, nil + } + return false, fmt.Errorf("Couldn't get cluster status: %v", err) + } + + log.Printf("Cluster '%v' status: %v", name, *clusterRes.Cluster.Status) + return false, nil +} + +// NodeGroupCreate creates a new k8s nodegroup in an existing cluster. +func (c *EKS) NodeGroupCreate(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + + for _, nodegroupReq := range req.NodeGroups { + nodegroupReq.ClusterName = req.Cluster.Name + log.Printf("Nodegroup create request: NodeGroupName: '%s', ClusterName: '%s'", *nodegroupReq.NodegroupName, *req.Cluster.Name) + _, err := c.clientEKS.CreateNodegroup(&nodegroupReq) + if err != nil { + return fmt.Errorf("Couldn't create nodegroup '%s' for cluster '%s', file:%v ,err: %v", *nodegroupReq.NodegroupName, *req.Cluster.Name, deployment.FileName, err) + } + + err = provider.RetryUntilTrue( + fmt.Sprintf("creating nodegroup:%s for cluster:%s", *nodegroupReq.NodegroupName, *req.Cluster.Name), + provider.GlobalRetryCount, + func() (bool, error) { return c.nodeGroupCreated(*nodegroupReq.NodegroupName, *req.Cluster.Name) }, + ) + + if err != nil { + return fmt.Errorf("creating nodegroup err:%v", err) + } + } + } + return nil +} + +// NodeGroupDelete deletes a k8s nodegroup in an existing cluster +func (c *EKS) NodeGroupDelete(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + + for _, nodegroupReq := range req.NodeGroups { + nodegroupReq.ClusterName = req.Cluster.Name + log.Printf("Nodegroup delete request: NodeGroupName: '%s', ClusterName: '%s'", *nodegroupReq.NodegroupName, *req.Cluster.Name) + reqD := eks.DeleteNodegroupInput{ + ClusterName: req.Cluster.Name, + NodegroupName: nodegroupReq.NodegroupName, + } + _, err := c.clientEKS.DeleteNodegroup(&reqD) + if err != nil { + return fmt.Errorf("Couldn't delete nodegroup '%s' for cluster '%s, file:%v ,err: %v", *nodegroupReq.NodegroupName, *req.Cluster.Name, deployment.FileName, err) + } + err = provider.RetryUntilTrue( + fmt.Sprintf("deleting nodegroup:%s for cluster:%s", *nodegroupReq.NodegroupName, *req.Cluster.Name), + provider.GlobalRetryCount, + func() (bool, error) { return c.nodeGroupDeleted(*nodegroupReq.NodegroupName, *req.Cluster.Name) }, + ) + + if err != nil { + return fmt.Errorf("deleting nodegroup err:%v", err) + } + + } + } + return nil +} + +func (c *EKS) nodeGroupCreated(nodegroupName, clusterName string) (bool, error) { + req := &eks.DescribeNodegroupInput{ + ClusterName: aws.String(clusterName), + NodegroupName: aws.String(nodegroupName), + } + nodegroupRes, err := c.clientEKS.DescribeNodegroup(req) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == eks.ErrCodeNotFoundException { + return false, nil + } + return false, fmt.Errorf("Couldn't get nodegroupname status: %v", err) + } + if *nodegroupRes.Nodegroup.Status == eks.NodegroupStatusActive { + return true, nil + } + + log.Printf("Nodegroup '%v' for Cluster '%v' status: %v", nodegroupName, clusterName, *nodegroupRes.Nodegroup.Status) + return false, nil + +} + +func (c *EKS) nodeGroupDeleted(nodegroupName, clusterName string) (bool, error) { + req := &eks.DescribeNodegroupInput{ + ClusterName: aws.String(clusterName), + NodegroupName: aws.String(nodegroupName), + } + nodegroupRes, err := c.clientEKS.DescribeNodegroup(req) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == eks.ErrCodeResourceNotFoundException { + return true, nil + } + return false, fmt.Errorf("Couldn't get nodegroupname status: %v", err) + } + + log.Printf("Nodegroup '%v' for Cluster '%v' status: %v", nodegroupName, clusterName, *nodegroupRes.Nodegroup.Status) + return false, nil +} + +// AllNodeGroupsRunning returns an error if at least one node pool is not running +func (c *EKS) AllNodeGroupsRunning(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + for _, nodegroup := range req.NodeGroups { + isRunning, err := c.nodeGroupCreated(*nodegroup.NodegroupName, *req.Cluster.Name) + if err != nil { + return fmt.Errorf("error fetching nodegroup info") + } + if !isRunning { + return fmt.Errorf("nodepool not running name: %v", *nodegroup.NodegroupName) + } + } + } + return nil +} + +// AllNodeGroupsDeleted returns an error if at least one node pool is not deleted +func (c *EKS) AllNodeGroupsDeleted(*kingpin.ParseContext) error { + req := &eksCluster{} + for _, deployment := range c.eksResources { + if err := yamlGo.UnmarshalStrict(deployment.Content, req); err != nil { + return fmt.Errorf("Error parsing the cluster deployment file %s:%v", deployment.FileName, err) + } + for _, nodegroup := range req.NodeGroups { + isRunning, err := c.nodeGroupDeleted(*nodegroup.NodegroupName, *req.Cluster.Name) + if err != nil { + return fmt.Errorf("error fetching nodegroup info") + } + if !isRunning { + return fmt.Errorf("nodepool not running name: %v", *nodegroup.NodegroupName) + } + } + } + return nil +} + +// EKSK8sToken returns aws iam authenticator token which is used to access eks k8s cluster from outside. +func (c *EKS) EKSK8sToken(clusterName, region string) awsToken.Token { + + gen, err := awsToken.NewGenerator(true, false) + + if err != nil { + log.Fatalf("Token abstraction error: %v", err) + } + + opts := &awsToken.GetTokenOptions{ + ClusterID: clusterName, + Session: c.sessionAWS, + } + + tok, err := gen.GetWithOptions(opts) + + if err != nil { + log.Fatalf("Token abstraction error: %v", err) + } + + return tok +} + +// NewK8sProvider sets the k8s provider used for deploying k8s manifests +func (c *EKS) NewK8sProvider(*kingpin.ParseContext) error { + + clusterName := c.DeploymentVars["CLUSTER_NAME"] + region := c.DeploymentVars["ZONE"] + + req := &eks.DescribeClusterInput{ + Name: &clusterName, + } + + rep, err := c.clientEKS.DescribeCluster(req) + if err != nil { + return fmt.Errorf("failed to get cluster details: %v", err) + } + + arnRole := *rep.Cluster.Arn + + caCert, err := base64.StdEncoding.DecodeString(*rep.Cluster.CertificateAuthority.Data) + if err != nil { + return fmt.Errorf("failed to decode certificate: %v", err.Error()) + } + + cluster := clientcmdapi.NewCluster() + cluster.CertificateAuthorityData = []byte(caCert) + cluster.Server = *rep.Cluster.Endpoint + + clusterContext := clientcmdapi.NewContext() + clusterContext.Cluster = arnRole + clusterContext.AuthInfo = arnRole + + authInfo := clientcmdapi.NewAuthInfo() + authInfo.Token = c.EKSK8sToken(clusterName, region).Token + + config := clientcmdapi.NewConfig() + config.AuthInfos[arnRole] = authInfo + config.Contexts[arnRole] = clusterContext + config.Clusters[arnRole] = cluster + config.CurrentContext = arnRole + config.Kind = "Config" + config.APIVersion = "v1" + + c.k8sProvider, err = k8sProvider.New(c.ctx, config) + if err != nil { + return fmt.Errorf("k8s provider error %v", err) + } + + return nil +} + +// ResourceApply calls k8s.ResourceApply to apply the k8s objects in the manifest files. +func (c *EKS) ResourceApply(*kingpin.ParseContext) error { + if err := c.k8sProvider.ResourceApply(c.k8sResources); err != nil { + return fmt.Errorf("error while applying a resource err: %v", err) + } + return nil +} + +// ResourceDelete calls k8s.ResourceDelete to apply the k8s objects in the manifest files. +func (c *EKS) ResourceDelete(*kingpin.ParseContext) error { + if err := c.k8sProvider.ResourceDelete(c.k8sResources); err != nil { + return fmt.Errorf("error while deleting objects from a manifest file err: %v", err) + } + return nil +} + +// GetDeploymentVars shows deployment variables. +func (c *EKS) GetDeploymentVars(*kingpin.ParseContext) error { + fmt.Print("-------------------\n DeploymentVars \n------------------- \n") + for key, value := range c.DeploymentVars { + fmt.Println(key, " : ", value) + } + + return nil +} diff --git a/pkg/provider/k8s/k8s.go b/pkg/provider/k8s/k8s.go index d7e70b1c2..6d18f3eb9 100644 --- a/pkg/provider/k8s/k8s.go +++ b/pkg/provider/k8s/k8s.go @@ -1280,7 +1280,15 @@ func (c *K8s) serviceExists(resource runtime.Object) (bool, error) { if len(res.Status.LoadBalancer.Ingress) > 0 { log.Printf("\tService %s Details", req.Name) for _, x := range res.Status.LoadBalancer.Ingress { - log.Printf("\t\thttp://%s:%d", x.IP, res.Spec.Ports[0].Port) + + ingressHostAddr := "" + if len(x.IP) != 0 { + ingressHostAddr = x.IP + } else { + ingressHostAddr = x.Hostname + } + + log.Printf("\t\thttp://%s:%d", ingressHostAddr, res.Spec.Ports[0].Port) } return true, nil } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 546b4bc7b..2910abb91 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -26,6 +26,7 @@ import ( ) const ( + EKSRetryCount = 100 GlobalRetryCount = 50 Separator = "---" globalRetryTime = 10 * time.Second @@ -47,8 +48,10 @@ func NewDeploymentResource() *DeploymentResource { DeploymentFiles: []string{}, FlagDeploymentVars: map[string]string{}, DefaultDeploymentVars: map[string]string{ - "NGINX_SERVICE_TYPE": "LoadBalancer", - "LOADGEN_SCALE_UP_REPLICAS": "10", + "NGINX_SERVICE_TYPE": "LoadBalancer", + "LOADGEN_SCALE_UP_REPLICAS": "10", + "SEPARATOR": ",", + "SERVICEACCOUNT_CLIENT_EMAIL": "example@example.com", }, } } @@ -79,11 +82,14 @@ func RetryUntilTrue(name string, retryCount int, fn func() (bool, error)) error func applyTemplateVars(content []byte, deploymentVars map[string]string) ([]byte, error) { fileContentParsed := bytes.NewBufferString("") t := template.New("resource").Option("missingkey=error") - // k8s objects can't have dots(.) se we add a custom function to allow normalising the variable values. t = t.Funcs(template.FuncMap{ + // k8s objects can't have dots(.) se we add a custom function to allow normalising the variable values. "normalise": func(t string) string { return strings.Replace(t, ".", "-", -1) }, + "split": func(rangeVars, separator string) []string { + return strings.Split(rangeVars, separator) + }, }) if err := template.Must(t.Parse(string(content))).Execute(fileContentParsed, deploymentVars); err != nil { return nil, fmt.Errorf("Failed to execute parse file err: %s", err) diff --git a/prombench/Makefile b/prombench/Makefile index 1ae2c5975..253db966e 100644 --- a/prombench/Makefile +++ b/prombench/Makefile @@ -3,43 +3,55 @@ INFRA_CMD ?= ../infra/infra PROVIDER ?= gke .PHONY: deploy clean -deploy: nodepool_create resource_apply -# GCP sometimes takes longer than 30 tries when trying to delete nodepools +deploy: node_create resource_apply +# GCP sometimes takes longer than 30 tries when trying to delete nodes # if k8s resources are not already cleared -clean: resource_delete nodepool_delete +clean: resource_delete node_delete -nodepool_create: - $(INFRA_CMD) $(PROVIDER) nodes create -a ${AUTH_FILE} \ - -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ - -f manifests/prombench/nodes_$(PROVIDER).yaml +node_create: + ${INFRA_CMD} ${PROVIDER} nodes create -a ${AUTH_FILE} \ + -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \ + -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ + -f manifests/prombench/nodes_${PROVIDER}.yaml resource_apply: - $(INFRA_CMD) $(PROVIDER) resource apply -a ${AUTH_FILE} \ - -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} \ + $(INFRA_CMD) ${PROVIDER} resource apply -a ${AUTH_FILE} \ + -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v CLUSTER_NAME:${CLUSTER_NAME} \ -v PR_NUMBER:${PR_NUMBER} -v RELEASE:${RELEASE} -v DOMAIN_NAME:${DOMAIN_NAME} \ -v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \ -f manifests/prombench/benchmark -# Required because namespace and cluster-role are not part of the created nodepools +# Required because namespace and cluster-role are not part of the created nodes resource_delete: - $(INFRA_CMD) $(PROVIDER) resource delete -a ${AUTH_FILE} \ - -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ + $(INFRA_CMD) ${PROVIDER} resource delete -a ${AUTH_FILE} \ + -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ -f manifests/prombench/benchmark/1c_cluster-role-binding.yaml \ -f manifests/prombench/benchmark/1a_namespace.yaml -nodepool_delete: - $(INFRA_CMD) $(PROVIDER) nodes delete -a ${AUTH_FILE} \ - -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ - -f manifests/prombench/nodes_$(PROVIDER).yaml +node_delete: + $(INFRA_CMD) ${PROVIDER} nodes delete -a ${AUTH_FILE} \ + -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \ + -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ + -f manifests/prombench/nodes_${PROVIDER}.yaml -all_nodepools_running: - $(INFRA_CMD) $(PROVIDER) nodes check-running -a ${AUTH_FILE} \ +all_nodes_running: + $(INFRA_CMD) ${PROVIDER} nodes check-running -a ${AUTH_FILE} \ -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \ -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ - -f manifests/prombench/nodes_$(PROVIDER).yaml + -f manifests/prombench/nodes_${PROVIDER}.yaml -all_nodepools_deleted: - $(INFRA_CMD) $(PROVIDER) nodes check-deleted -a ${AUTH_FILE} \ +all_nodes_deleted: + $(INFRA_CMD) ${PROVIDER} nodes check-deleted -a ${AUTH_FILE} \ -v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \ + -v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \ + -v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \ -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \ - -f manifests/prombench/nodes_$(PROVIDER).yaml + -f manifests/prombench/nodes_${PROVIDER}.yaml diff --git a/prombench/README.md b/prombench/README.md index e60c71854..56fe477bf 100644 --- a/prombench/README.md +++ b/prombench/README.md @@ -8,8 +8,8 @@ It is designed to support adding more k8s providers. ## Overview of the manifest files The `/manifest` directory contains all the kubernetes manifest files. - - `cluster_gke.yaml` : This is used to create the Main Node in gke. +- `cluster_eks.yaml` : This is used to create the Main Node in eks. - `cluster-infra/` : These are the persistent components of the Main Node. - `prombench/` : These resources are created and destroyed for each prombench test. @@ -19,6 +19,7 @@ Prombench can be run on various providers, following are the provider specific i - Instructions for [Google Kubernetes Engine](docs/gke.md) - Instructions for [Kubernetes In Docker](docs/kind.md) +- Instructions for [Elastic Kubernetes Service](docs/eks.md) ## Setup GitHub Actions diff --git a/prombench/docs/eks.md b/prombench/docs/eks.md new file mode 100644 index 000000000..87a548251 --- /dev/null +++ b/prombench/docs/eks.md @@ -0,0 +1,113 @@ +# Prombench in EKS + +Run prombench tests in [Elastic Kubernetes Service](https://aws.amazon.com/eks/). + +## Setup prombench + +1. [Create the main node](#create-the-main-node) +2. [Deploy monitoring components](#deploy-monitoring-components) + +### Create the Main Node + +--- + +- Create [security credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) on AWS. Put the credentials into a `yaml` file as follows: +```yaml +accesskeyid: +secretaccesskey: +``` +- Create a [VPC](https://docs.aws.amazon.com/eks/latest/userguide/create-public-private-vpc.html) with public subnets. +- Create a [Amazon EKS cluster role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) with following policies: + - AmazonEKSclusterPolicy +- Create a [Amazon EKS worker node role](https://docs.aws.amazon.com/eks/latest/userguide/worker_node_IAM_role.html) with following policies: + - AmazonEKSWorkerNodePolicy, AmazonEKS_CNI_Policy, AmazonEC2ContainerRegistryReadOnly +- Set the following environment variables and deploy the cluster. + +```shell +export AUTH_FILE= +export CLUSTER_NAME=prombench +export ZONE=us-east-1 +export EKS_WORKER_ROLE_ARN= +export EKS_CLUSTER_ROLE_ARN= +# By default SEPARATOR DeploymentVar is set to `,` but you can override it by exporting and +# then passing it with the -v flag. It is used to split DeploymentVar into a slice. +export SEPARATOR=, +export EKS_SUBNET_IDS=SUBNETID1,SUBNETID2,SUBNETID3 + +../infra/infra eks cluster create -a $AUTH_FILE -v ZONE:$ZONE \ + -v EKS_WORKER_ROLE_ARN:$EKS_WORKER_ROLE_ARN -v EKS_CLUSTER_ROLE_ARN:$EKS_CLUSTER_ROLE_ARN \ + -v EKS_SUBNET_IDS:$EKS_SUBNET_IDS \ + -v CLUSTER_NAME:$CLUSTER_NAME \ + -f manifests/cluster_eks.yaml +``` + + +### Deploy monitoring components + + +> Collecting, monitoring and displaying the test results and logs +--- + +- [Optional] If used with the Github integration generate a GitHub auth token. + - Login with the [Prombot account](https://github.com/prombot) and generate a [new auth token](https://github.com/settings/tokens). + - With permissions: `public_repo`, `read:org`, `write:discussion`. + +```shell +export GRAFANA_ADMIN_PASSWORD=password +export DOMAIN_NAME=prombench.prometheus.io // Can be set to any other custom domain or an empty string when not used with the Github integration. +export OAUTH_TOKEN= +export WH_SECRET= +export GITHUB_ORG=prometheus +export GITHUB_REPO=prometheus +``` + +- Deploy the [nginx-ingress-controller](https://github.com/kubernetes/ingress-nginx), Prometheus-Meta, Loki, Grafana, Alertmanager & Github Notifier. + +```shell +../infra/infra eks resource apply -a $AUTH_FILE -v ZONE:$ZONE \ + -v CLUSTER_NAME:$CLUSTER_NAME -v DOMAIN_NAME:$DOMAIN_NAME \ + -v GRAFANA_ADMIN_PASSWORD:$GRAFANA_ADMIN_PASSWORD \ + -v OAUTH_TOKEN="$(printf $OAUTH_TOKEN | base64 -w 0)" \ + -v WH_SECRET="$(printf $WH_SECRET | base64 -w 0)" \ + -v GITHUB_ORG:$GITHUB_ORG -v GITHUB_REPO:$GITHUB_REPO \ + -f manifests/cluster-infra +``` + +- The output will show the ingress IP which will be used to point the domain name to. +- Set the `A record` for `` to point to `nginx-ingress-controller` IP address. +- The services will be accessible at: + - Grafana :: `http:///grafana` + - Prometheus :: `http:///prometheus-meta` + - Logs :: `http:///grafana/explore` + +## Usage + +### Start a benchmarking test manually +--- + +- Set the following environment variables. + +```shell +export RELEASE= +export PR_NUMBER= +``` + +- Create the nodegroups for the k8s objects + +```shell +../infra/infra eks nodes create -a $AUTH_FILE \ + -v ZONE:$ZONE -v EKS_WORKER_ROLE_ARN:$EKS_WORKER_ROLE_ARN -v EKS_CLUSTER_ROLE_ARN:$EKS_CLUSTER_ROLE_ARN \ + -v EKS_SUBNET_IDS:$EKS_SUBNET_IDS \ + -v CLUSTER_NAME:$CLUSTER_NAME \ + -v PR_NUMBER:$PR_NUMBER -f manifests/prombench/nodes_eks.yaml +``` + +- Deploy the k8s objects + +```shell +../infra/infra eks resource apply -a $AUTH_FILE \ + -v ZONE:$ZONE -v CLUSTER_NAME:$CLUSTER_NAME \ + -v PR_NUMBER:$PR_NUMBER -v RELEASE:$RELEASE -v DOMAIN_NAME:$DOMAIN_NAME \ + -v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \ + -f manifests/prombench/benchmark +``` diff --git a/prombench/manifests/cluster_eks.yaml b/prombench/manifests/cluster_eks.yaml new file mode 100644 index 000000000..cbe3326b0 --- /dev/null +++ b/prombench/manifests/cluster_eks.yaml @@ -0,0 +1,26 @@ +cluster: + name: {{ .CLUSTER_NAME }} + version: 1.14 + rolearn: {{ .EKS_CLUSTER_ROLE_ARN }} + resourcesvpcconfig: + endpointpublicaccess: true + subnetids: + {{ range $subnetId := split .EKS_SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} +nodegroups: + - nodegroupname: main-node + noderole: {{ .EKS_WORKER_ROLE_ARN }} + disksize: 300 + subnets: + {{ range $subnetId := split .EKS_SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} + instancetypes: + - t3.xlarge + scalingconfig: + desiredsize: 1 + maxsize: 1 + minsize: 1 + labels: + node-name: main-node diff --git a/prombench/manifests/prombench/nodes_eks.yaml b/prombench/manifests/prombench/nodes_eks.yaml new file mode 100644 index 000000000..a73e9ba00 --- /dev/null +++ b/prombench/manifests/prombench/nodes_eks.yaml @@ -0,0 +1,35 @@ +cluster: + name: {{ .CLUSTER_NAME }} +nodegroups: + - nodegroupname: prometheus-{{ .PR_NUMBER }} + noderole: {{ .EKS_WORKER_ROLE_ARN }} + disksize: 100 + subnets: + {{ range $subnetId := split .EKS_SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} + instancetypes: + - r5d.2xlarge #This machine has SSD. SSD is used to give fast-lookup to Prometheus servers being benchmarked + scalingconfig: + desiredsize: 2 + maxsize: 2 + minsize: 2 + labels: + isolation: prometheus + node-name: prometheus-{{ .PR_NUMBER }} + - nodegroupname: nodes-{{ .PR_NUMBER }} + noderole: {{ .EKS_WORKER_ROLE_ARN }} + disksize: 100 + subnets: + {{ range $subnetId := split .EKS_SUBNET_IDS .SEPARATOR }} + - {{ $subnetId }} + {{ end }} + instancetypes: + - c5.4xlarge + scalingconfig: + desiredsize: 1 + maxsize: 1 + minsize: 1 + labels: + isolation: none + node-name: nodes-{{ .PR_NUMBER }}