From 4ea39257dbffcae8d3a6a457e1cb43ebd77a8f73 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Sat, 9 Sep 2023 09:02:59 +0800 Subject: [PATCH] feat: kbcli cluster register cmd for cluster create config (#5052) Co-authored-by: 1aal <1aal@users.noreply.github.com> --- Makefile | 2 +- deploy/orioledb-cluster/values.schema.json | 2 +- docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_cluster.md | 1 + docs/user_docs/cli/kbcli_cluster_create.md | 1 - internal/cli/cluster/builtin_charts.go | 122 +++++++++++++ internal/cli/cluster/cluster_chart.go | 47 +---- internal/cli/cluster/external_charts.go | 194 +++++++++++++++++++++ internal/cli/cluster/kafka.go | 34 ---- internal/cli/cluster/mongodb.go | 34 ---- internal/cli/cluster/mysql.go | 34 ---- internal/cli/cluster/neon.go | 15 -- internal/cli/cluster/postgresql.go | 34 ---- internal/cli/cluster/redis.go | 34 ---- internal/cli/cluster/register.go | 46 +++++ internal/cli/cluster/register_test.go | 100 +++++++++++ internal/cli/cmd/cluster/cluster.go | 1 + internal/cli/cmd/cluster/create_subcmds.go | 7 +- internal/cli/cmd/cluster/register.go | 116 ++++++++++++ internal/cli/cmd/cluster/register_test.go | 61 +++++++ internal/cli/types/types.go | 7 + internal/cli/util/helm/downloader.go | 60 +++++++ 22 files changed, 724 insertions(+), 229 deletions(-) create mode 100644 internal/cli/cluster/builtin_charts.go create mode 100644 internal/cli/cluster/external_charts.go delete mode 100644 internal/cli/cluster/kafka.go delete mode 100644 internal/cli/cluster/mongodb.go delete mode 100644 internal/cli/cluster/mysql.go delete mode 100644 internal/cli/cluster/neon.go delete mode 100644 internal/cli/cluster/postgresql.go delete mode 100644 internal/cli/cluster/redis.go create mode 100644 internal/cli/cluster/register.go create mode 100644 internal/cli/cluster/register_test.go create mode 100644 internal/cli/cmd/cluster/register.go create mode 100644 internal/cli/cmd/cluster/register_test.go create mode 100644 internal/cli/util/helm/downloader.go diff --git a/Makefile b/Makefile index 45fd15511f6..3fc13cdf5d1 100644 --- a/Makefile +++ b/Makefile @@ -302,7 +302,7 @@ build-kbcli-embed-chart: helmtool create-kbcli-embed-charts-dir \ build-single-kbcli-embed-chart.postgresql-cluster \ build-single-kbcli-embed-chart.kafka-cluster \ build-single-kbcli-embed-chart.mongodb-cluster \ - build-single-kbcli-embed-chart.neon-cluster +# build-single-kbcli-embed-chart.neon-cluster # build-single-kbcli-embed-chart.postgresql-cluster \ # build-single-kbcli-embed-chart.clickhouse-cluster \ # build-single-kbcli-embed-chart.milvus-cluster \ diff --git a/deploy/orioledb-cluster/values.schema.json b/deploy/orioledb-cluster/values.schema.json index dc8497b2fb1..7bd519f63f8 100644 --- a/deploy/orioledb-cluster/values.schema.json +++ b/deploy/orioledb-cluster/values.schema.json @@ -6,7 +6,7 @@ "title": "Version", "description": "Cluster version.", "type": "string", - "default": "postgresql-14.8.0" + "default": "orioledb-beta1" }, "mode": { "title": "Mode", diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 0c09f529411..d5131af3a02 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -101,6 +101,7 @@ Cluster command. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. * [kbcli cluster promote](kbcli_cluster_promote.md) - Promote a non-primary or non-leader instance as the new primary or leader of the cluster +* [kbcli cluster register](kbcli_cluster_register.md) - Pull the cluster chart to the local cache and register the type to 'create' sub-command * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. * [kbcli cluster revoke-role](kbcli_cluster_revoke-role.md) - Revoke role from account diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index fdd17612564..3b89445f452 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -71,6 +71,7 @@ Cluster command. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. * [kbcli cluster promote](kbcli_cluster_promote.md) - Promote a non-primary or non-leader instance as the new primary or leader of the cluster +* [kbcli cluster register](kbcli_cluster_register.md) - Pull the cluster chart to the local cache and register the type to 'create' sub-command * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. * [kbcli cluster revoke-role](kbcli_cluster_revoke-role.md) - Revoke role from account diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 037baf557b0..22d2d8692b3 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -148,7 +148,6 @@ kbcli cluster create [NAME] [flags] * [kbcli cluster create kafka](kbcli_cluster_create_kafka.md) - Create a kafka cluster. * [kbcli cluster create mongodb](kbcli_cluster_create_mongodb.md) - Create a mongodb cluster. * [kbcli cluster create mysql](kbcli_cluster_create_mysql.md) - Create a mysql cluster. -* [kbcli cluster create neon](kbcli_cluster_create_neon.md) - Create a neon cluster. * [kbcli cluster create postgresql](kbcli_cluster_create_postgresql.md) - Create a postgresql cluster. * [kbcli cluster create redis](kbcli_cluster_create_redis.md) - Create a redis cluster. diff --git a/internal/cli/cluster/builtin_charts.go b/internal/cli/cluster/builtin_charts.go new file mode 100644 index 00000000000..4dfeff7ae20 --- /dev/null +++ b/internal/cli/cluster/builtin_charts.go @@ -0,0 +1,122 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "embed" + "fmt" + "io" +) + +// embedConfig is the interface for the go embed chart +type embedConfig struct { + chartFS embed.FS + // chart file name, include the extension + name string + // chart alias, this alias will be used as the command alias + alias string +} + +var _ chartLoader = &embedConfig{} + +func (e *embedConfig) register(subcmd ClusterType) error { + if _, ok := ClusterTypeCharts[subcmd]; ok { + return fmt.Errorf("cluster type %s already registered", subcmd) + } + ClusterTypeCharts[subcmd] = e + return nil +} + +func (e *embedConfig) getAlias() string { + return e.alias +} + +func (e *embedConfig) loadChart() (io.ReadCloser, error) { + return e.chartFS.Open(fmt.Sprintf("charts/%s", e.name)) +} + +func (e *embedConfig) getChartFileName() string { + return e.name +} + +var ( + // run `make generate` to generate this embed file + //go:embed charts/apecloud-mysql-cluster.tgz + mysqlChart embed.FS + //go:embed charts/postgresql-cluster.tgz + postgresqlChart embed.FS + //go:embed charts/kafka-cluster.tgz + kafkaChart embed.FS + //go:embed charts/redis-cluster.tgz + redisChart embed.FS + //go:embed charts/mongodb-cluster.tgz + mongodbChart embed.FS +) + +// internal_chart registers embed chart + +func init() { + + mysql := &embedConfig{ + chartFS: mysqlChart, + name: "apecloud-mysql-cluster.tgz", + alias: "", + } + if err := mysql.register("mysql"); err != nil { + fmt.Println(err.Error()) + } + + postgresql := &embedConfig{ + chartFS: postgresqlChart, + name: "postgresql-cluster.tgz", + alias: "", + } + if err := postgresql.register("postgresql"); err != nil { + fmt.Println(err.Error()) + } + + kafka := &embedConfig{ + chartFS: kafkaChart, + name: "kafka-cluster.tgz", + alias: "", + } + if err := kafka.register("kafka"); err != nil { + fmt.Println(err.Error()) + } + + redis := &embedConfig{ + chartFS: redisChart, + name: "redis-cluster.tgz", + alias: "", + } + if err := redis.register("redis"); err != nil { + fmt.Println(err.Error()) + } + + mongodb := &embedConfig{ + chartFS: mongodbChart, + name: "mongodb-cluster.tgz", + alias: "", + } + if err := mongodb.register("mongodb"); err != nil { + fmt.Println(err.Error()) + } + +} diff --git a/internal/cli/cluster/cluster_chart.go b/internal/cli/cluster/cluster_chart.go index 1c900d58166..2d17d97550b 100644 --- a/internal/cli/cluster/cluster_chart.go +++ b/internal/cli/cluster/cluster_chart.go @@ -21,7 +21,6 @@ package cluster import ( "compress/gzip" - "embed" "fmt" "strings" @@ -74,33 +73,6 @@ type ChartInfo struct { Alias string } -type ( - // ClusterType is the type of the cluster - ClusterType string - - // chartConfig is the helm chart config - chartConfig struct { - chartFS embed.FS - - // chart file name, include the extension - name string - - // chart alias, this alias will be used as the command alias - alias string - } -) - -var clusterTypeCharts = map[ClusterType]chartConfig{} - -// registerClusterType registers the cluster type, the ClusterType t will be used as -// the command name, the alias will be used as the command alias. -func registerClusterType(t ClusterType, chartFS embed.FS, name string, alias string) { - if _, ok := clusterTypeCharts[t]; ok { - panic(fmt.Sprintf("cluster type %s already registered", t)) - } - clusterTypeCharts[t] = chartConfig{chartFS: chartFS, name: name, alias: alias} -} - func BuildChartInfo(t ClusterType) (*ChartInfo, error) { var err error @@ -248,12 +220,11 @@ func ValidateValues(c *ChartInfo, values map[string]interface{}) error { } func loadHelmChart(ci *ChartInfo, t ClusterType) error { - cf, ok := clusterTypeCharts[t] + cf, ok := ClusterTypeCharts[t] if !ok { return fmt.Errorf("failed to find the helm chart of %s", t) } - - file, err := cf.chartFS.Open(fmt.Sprintf("charts/%s", cf.name)) + file, err := cf.loadChart() if err != nil { return err } @@ -262,7 +233,7 @@ func loadHelmChart(ci *ChartInfo, t ClusterType) error { c, err := loader.LoadArchive(file) if err != nil { if err == gzip.ErrHeader { - return fmt.Errorf("file '%s' does not appear to be a valid chart file (details: %s)", cf.name, err) + return fmt.Errorf("file '%s' does not appear to be a valid chart file (details: %s)", cf.getChartFileName(), err) } } @@ -271,22 +242,18 @@ func loadHelmChart(ci *ChartInfo, t ClusterType) error { } ci.Chart = c - ci.Alias = cf.alias + ci.Alias = cf.getAlias() return nil } func SupportedTypes() []ClusterType { - types := maps.Keys(clusterTypeCharts) - slices.SortFunc(maps.Keys(clusterTypeCharts), func(i, j ClusterType) bool { - return i < j + types := maps.Keys(ClusterTypeCharts) + slices.SortFunc(types, func(i, j ClusterType) bool { + return i.String() < j.String() }) return types } -func (t ClusterType) String() string { - return string(t) -} - func (s SchemaPropName) String() string { return string(s) } diff --git a/internal/cli/cluster/external_charts.go b/internal/cli/cluster/external_charts.go new file mode 100644 index 00000000000..6835ba3906b --- /dev/null +++ b/internal/cli/cluster/external_charts.go @@ -0,0 +1,194 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "gopkg.in/yaml.v2" + "k8s.io/klog" + + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +// CliClusterChartConfig is $HOME/.kbcli/cluster_types by default +var CliClusterChartConfig string + +// CliChartsCacheDir is $HOME/.kbcli/charts by default +var CliChartsCacheDir string + +type clusterConfig []*TypeInstance + +// GlobalClusterChartConfig is kbcli global cluster chart config reference to CliClusterChartConfig +var GlobalClusterChartConfig clusterConfig +var CacheFiles []fs.DirEntry + +// ReadConfigs read the config from configPath +func (c *clusterConfig) ReadConfigs(configPath string) error { + contents, err := os.ReadFile(configPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + contents = []byte{} + } + err = yaml.Unmarshal(contents, c) + if err != nil { + return err + } + return nil +} + +// WriteConfigs write current config into configPath +func (c *clusterConfig) WriteConfigs(configPath string) error { + newConfig, err := yaml.Marshal(*c) + if err != nil { + return err + } + return os.WriteFile(configPath, newConfig, 0666) +} + +// AddConfig add a new cluster type instance into current config +func (c *clusterConfig) AddConfig(add *TypeInstance) { + *c = append(*c, add) +} + +// RemoveConfig remove a ClusterType from current config +func (c *clusterConfig) RemoveConfig(name ClusterType) bool { + tempList := *c + for i, chart := range tempList { + if chart.Name == name { + *c = append((*c)[:i], (*c)[i+1:]...) + return true + } + } + return false +} +func (c *clusterConfig) Len() int { + return len(*c) +} + +// RegisterCMD will register all cluster type instances in the config c and auto clear the register failed instances +// and rewrite config +func RegisterCMD(c clusterConfig, configPath string) { + var needRemove []ClusterType + for _, config := range c { + if err := config.register(config.Name); err != nil { + klog.V(2).Info(err.Error()) + needRemove = append(needRemove, config.Name) + } + } + for _, name := range needRemove { + c.RemoveConfig(name) + } + if err := c.WriteConfigs(configPath); err != nil { + klog.V(2).Info(fmt.Sprintf("Warning: auto clear kbcli cluster chart config failed %s\n", err.Error())) + } +} + +func GetChartCacheFiles() []fs.DirEntry { + homeDir, _ := util.GetCliHomeDir() + dirFS := os.DirFS(homeDir) + result, err := fs.ReadDir(dirFS, "charts") + if err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(CliChartsCacheDir, 0777) + if err != nil { + klog.V(2).Info(fmt.Sprintf("Failed to create charts cache dir %s: %s", CliChartsCacheDir, err.Error())) + return nil + } + result = []fs.DirEntry{} + } else { + klog.V(2).Info(fmt.Sprintf("Failed to create charts cache dir %s: %s", CliChartsCacheDir, err.Error())) + return nil + } + } + return result +} + +func ClearCharts(c ClusterType) { + // if the fail clusterType is from external config, remove the config and the elated charts + if GlobalClusterChartConfig.RemoveConfig(c) { + if err := GlobalClusterChartConfig.WriteConfigs(CliClusterChartConfig); err != nil { + klog.V(2).Info(fmt.Sprintf("Warning: auto clear %s config fail due to: %s\n", c, err.Error())) + + } + if err := os.Remove(filepath.Join(CliChartsCacheDir, ClusterTypeCharts[c].getChartFileName())); err != nil { + klog.V(2).Info(fmt.Sprintf("Warning: auto clear %s config fail due to: %s\n", c, err.Error())) + } + CacheFiles = GetChartCacheFiles() + } +} + +// TypeInstance reference to a cluster type instance in config +type TypeInstance struct { + Name ClusterType `yaml:"name"` + URL string `yaml:"helmChartUrl"` + Alias string `yaml:"alias"` +} + +func (h *TypeInstance) loadChart() (io.ReadCloser, error) { + return os.Open(filepath.Join(CliChartsCacheDir, h.getChartFileName())) +} + +func (h *TypeInstance) getChartFileName() string { + return path.Base(h.URL) +} + +func (h *TypeInstance) getAlias() string { + return h.Alias +} + +func (h *TypeInstance) register(subcmd ClusterType) error { + if _, ok := ClusterTypeCharts[subcmd]; ok { + return fmt.Errorf("cluster type %s already registered", subcmd) + } + ClusterTypeCharts[subcmd] = h + + for _, f := range CacheFiles { + if f.Name() == h.getChartFileName() { + return nil + } + } + return fmt.Errorf("can't find the %s in cache, please use 'kbcli cluster pull %s --url %s' first", h.Name.String(), h.Name.String(), h.URL) +} + +var _ chartLoader = &TypeInstance{} + +func init() { + homeDir, _ := util.GetCliHomeDir() + CliClusterChartConfig = filepath.Join(homeDir, types.CliClusterTypeConfigs) + CliChartsCacheDir = filepath.Join(homeDir, types.CliChartsCache) + + err := GlobalClusterChartConfig.ReadConfigs(CliClusterChartConfig) + if err != nil { + fmt.Println(err.Error()) + return + } + // check charts cache dir + CacheFiles = GetChartCacheFiles() + RegisterCMD(GlobalClusterChartConfig, CliClusterChartConfig) +} diff --git a/internal/cli/cluster/kafka.go b/internal/cli/cluster/kafka.go deleted file mode 100644 index 9f254a9d431..00000000000 --- a/internal/cli/cluster/kafka.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/kafka-cluster.tgz - kafkaChart embed.FS -) - -func init() { - registerClusterType("kafka", kafkaChart, "kafka-cluster.tgz", "") -} diff --git a/internal/cli/cluster/mongodb.go b/internal/cli/cluster/mongodb.go deleted file mode 100644 index 940b0cdafe2..00000000000 --- a/internal/cli/cluster/mongodb.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/mongodb-cluster.tgz - mongodbChart embed.FS -) - -func init() { - registerClusterType("mongodb", mongodbChart, "mongodb-cluster.tgz", "") -} diff --git a/internal/cli/cluster/mysql.go b/internal/cli/cluster/mysql.go deleted file mode 100644 index 867c2f634c0..00000000000 --- a/internal/cli/cluster/mysql.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/apecloud-mysql-cluster.tgz - mysqlChart embed.FS -) - -func init() { - registerClusterType("mysql", mysqlChart, "apecloud-mysql-cluster.tgz", "") -} diff --git a/internal/cli/cluster/neon.go b/internal/cli/cluster/neon.go deleted file mode 100644 index d77de839e68..00000000000 --- a/internal/cli/cluster/neon.go +++ /dev/null @@ -1,15 +0,0 @@ -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/neon-cluster.tgz - neonChart embed.FS -) - -func init() { - registerClusterType("neon", neonChart, "neon-cluster.tgz", "neon") -} diff --git a/internal/cli/cluster/postgresql.go b/internal/cli/cluster/postgresql.go deleted file mode 100644 index 92be90408ee..00000000000 --- a/internal/cli/cluster/postgresql.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/postgresql-cluster.tgz - postgresqlChart embed.FS -) - -func init() { - registerClusterType("postgresql", postgresqlChart, "postgresql-cluster.tgz", "pg") -} diff --git a/internal/cli/cluster/redis.go b/internal/cli/cluster/redis.go deleted file mode 100644 index d45d6d6832c..00000000000 --- a/internal/cli/cluster/redis.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cluster - -import ( - "embed" -) - -var ( - // run `make generate` to generate this embed file - //go:embed charts/redis-cluster.tgz - redisChart embed.FS -) - -func init() { - registerClusterType("redis", redisChart, "redis-cluster.tgz", "") -} diff --git a/internal/cli/cluster/register.go b/internal/cli/cluster/register.go new file mode 100644 index 00000000000..aff84165d39 --- /dev/null +++ b/internal/cli/cluster/register.go @@ -0,0 +1,46 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "io" +) + +type ClusterType string + +func (t ClusterType) String() string { + return string(t) +} + +type chartLoader interface { + // loadChart loads the chart content during building sub-command + loadChart() (io.ReadCloser, error) + // getChartFileName returns the chart file name, include the extension + getChartFileName() string + // getAlias returns the chart alias, this alias will be used as the command alias + getAlias() string + // register registers the cluster type as a sub cmd into create cluster command + register(subcmd ClusterType) error +} + +// ClusterTypeCharts is the map of the cluster type and the chart config +// ClusterType is the type of the cluster, the ClusterType t will be used as sub command name, +// chartLoader is the interface for the chart config, implement this interface to register cluster type. +var ClusterTypeCharts = map[ClusterType]chartLoader{} diff --git a/internal/cli/cluster/register_test.go b/internal/cli/cluster/register_test.go new file mode 100644 index 00000000000..f49b6e29ecd --- /dev/null +++ b/internal/cli/cluster/register_test.go @@ -0,0 +1,100 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "io" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("cluster register", func() { + It("test builtin chart", func() { + mysql := &embedConfig{ + chartFS: mysqlChart, + name: "apecloud-mysql-cluster.tgz", + alias: "", + } + Expect(mysql.register("mysql")).Should(HaveOccurred()) + Expect(mysql.register("mysql-other")).Should(Succeed()) + Expect(mysql.getChartFileName()).Should(Equal("apecloud-mysql-cluster.tgz")) + Expect(mysql.getAlias()).Should(Equal("")) + chart, err := mysql.loadChart() + Expect(err).Should(Succeed()) + bytes, err := io.ReadAll(chart) + Expect(bytes).ShouldNot(BeEmpty()) + Expect(err).Should(Succeed()) + }) + + It("test external chart", func() { + fakeChart := &TypeInstance{ + Name: "fake", + URL: "www.fake-chart-hub/fake.tgz", + Alias: "", + } + Expect(fakeChart.getAlias()).Should(Equal("")) + Expect(fakeChart.getChartFileName()).Should(Equal("fake.tgz")) + _, err := fakeChart.loadChart() + Expect(err).Should(HaveOccurred()) + Expect(fakeChart.register("fake")).Should(HaveOccurred()) + }) + + Context("test Config reader", func() { + var tempConfigPath string + + var tempCLusterConfig clusterConfig + var configContent = `- name: orioledb + helmChartUrl: https://github.com/apecloud/helm-charts/releases/download/orioledb-cluster-0.7.0-alpha.7/orioledb-cluster-0.7.0-alpha.7.tgz + alias: "" +` + BeforeEach(func() { + tempConfigPath = filepath.Join(os.TempDir(), "kbcli_test") + Expect(os.WriteFile(tempConfigPath, []byte(configContent), 0666)).Should(Succeed()) + }) + + AfterEach(func() { + os.Remove(tempConfigPath) + }) + + It("test read configs and remove", func() { + Expect(tempCLusterConfig.ReadConfigs(tempConfigPath)).Should(Succeed()) + Expect(tempCLusterConfig.Len()).Should(Equal(1)) + Expect(tempCLusterConfig.RemoveConfig("orioledb")).Should(BeTrue()) + Expect(tempCLusterConfig.Len()).Should(Equal(0)) + }) + + It("test add config and write", func() { + tempCLusterConfig.AddConfig(&TypeInstance{ + Name: "orioledb", + URL: "https://fakeurl.com", + Alias: "", + }) + Expect(tempCLusterConfig.Len()).Should(Equal(1)) + Expect(tempCLusterConfig.WriteConfigs(tempConfigPath)).Should(Succeed()) + + file, _ := os.ReadFile(tempConfigPath) + Expect(string(file)).Should(Equal("- name: orioledb\n helmChartUrl: https://fakeurl.com\n alias: \"\"\n")) + }) + + }) +}) diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index 6c53ae0291d..6ffde3d5249 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -56,6 +56,7 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr NewListEventsCmd(f, streams), NewLabelCmd(f, streams), NewDeleteCmd(f, streams), + newRegisterCmd(f, streams), }, }, { diff --git a/internal/cli/cmd/cluster/create_subcmds.go b/internal/cli/cmd/cluster/create_subcmds.go index 971e2a15f69..b2321c9225f 100644 --- a/internal/cli/cmd/cluster/create_subcmds.go +++ b/internal/cli/cmd/cluster/create_subcmds.go @@ -22,6 +22,7 @@ package cluster import ( "context" "fmt" + "os" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,7 +74,11 @@ func buildCreateSubCmds(createOptions *create.CreateOptions) []*cobra.Command { for _, t := range cluster.SupportedTypes() { o, err := newSubCmdsOptions(createOptions, t) - util.CheckErr(err) + if err != nil { + fmt.Fprintf(os.Stdout, "Failed add '%s' to 'create' sub command due to %s\n", t.String(), err.Error()) + cluster.ClearCharts(t) + continue + } cmd := &cobra.Command{ Use: t.String() + " NAME", diff --git a/internal/cli/cmd/cluster/register.go b/internal/cli/cmd/cluster/register.go new file mode 100644 index 00000000000..c6162588928 --- /dev/null +++ b/internal/cli/cmd/cluster/register.go @@ -0,0 +1,116 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "fmt" + "path/filepath" + "regexp" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/cluster" + "github.com/apecloud/kubeblocks/internal/cli/util/helm" +) + +var clusterRegisterExample = templates.Examples(` + # Pull a cluster type to local and register it to "kbcli cluster create" sub-cmd from specified URL + kbcli cluster register orioledb --url https://github.com/apecloud/helm-charts/releases/download/orioledb-cluster-0.6.0-beta.44/orioledb-cluster-0.6.0-beta.44.tgz +`) + +type registerOption struct { + Factory cmdutil.Factory + genericclioptions.IOStreams + + clusterType cluster.ClusterType + url string + alias string +} + +func newRegisterOption(f cmdutil.Factory, streams genericclioptions.IOStreams) *registerOption { + o := ®isterOption{ + Factory: f, + IOStreams: streams, + } + return o +} + +func newRegisterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := newRegisterOption(f, streams) + cmd := &cobra.Command{ + Use: "register [NAME] --url [CHART-URL]", + Short: "Pull the cluster chart to the local cache and register the type to 'create' sub-command", + Example: clusterRegisterExample, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + o.clusterType = cluster.ClusterType(args[0]) + cmdutil.CheckErr(o.validate()) + cmdutil.CheckErr(o.run()) + }, + } + cmd.Flags().StringVar(&o.url, "url", "", "Specify the cluster type chart download url") + cmd.Flags().StringVar(&o.alias, "alias", "", "Set the cluster type alias") + _ = cmd.MarkFlagRequired("url") + return cmd +} + +// validate will check the +func (o *registerOption) validate() error { + re := regexp.MustCompile(`^[a-zA-Z0-9]{1,9}$`) + if !re.MatchString(o.clusterType.String()) { + return fmt.Errorf("cluster type %s is not appropriate as a subcommand", o.clusterType.String()) + } + + for key := range cluster.ClusterTypeCharts { + if key == o.clusterType { + return fmt.Errorf("cluster type %s is already existed", o.clusterType.String()) + } + } + + return nil +} + +func (o *registerOption) run() error { + chartsDownloader, err := helm.NewDownloader(helm.NewConfig("default", "", "", false)) + if err != nil { + return err + } + // before download, we should check the chart name whether conflict in local cache + for _, file := range cluster.CacheFiles { + if file.Name() == filepath.Base(o.url) { + return fmt.Errorf("cluster type '%s' register failed due to the cluster chart's name conflict %s", o.clusterType, file.Name()) + } + } + + _, _, err = chartsDownloader.DownloadTo(o.url, "", cluster.CliChartsCacheDir) + if err != nil { + return err + } + cluster.GlobalClusterChartConfig.AddConfig(&cluster.TypeInstance{ + Name: o.clusterType, + URL: o.url, + Alias: o.alias, + }) + return cluster.GlobalClusterChartConfig.WriteConfigs(cluster.CliClusterChartConfig) + +} diff --git a/internal/cli/cmd/cluster/register_test.go b/internal/cli/cmd/cluster/register_test.go new file mode 100644 index 00000000000..77e47add76a --- /dev/null +++ b/internal/cli/cmd/cluster/register_test.go @@ -0,0 +1,61 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("cluster register", func() { + var streams genericclioptions.IOStreams + var tf *cmdtesting.TestFactory + + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace("default") + }) + + It("register command", func() { + option := newRegisterOption(tf, streams) + Expect(option).ShouldNot(BeNil()) + + cmd := newRegisterCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + + It("register command validate", func() { + o := ®isterOption{ + Factory: tf, + IOStreams: streams, + clusterType: "not-allow-name", + } + Expect(o.validate()).Should(HaveOccurred()) + + o.clusterType = "mysql" + // already exist + Expect(o.validate()).Should(HaveOccurred()) + + o.clusterType = "oracle" + Expect(o.validate()).Should(Succeed()) + }) +}) diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 785d78d48b1..328b1b91a0d 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -34,6 +34,13 @@ import ( const ( // CliDefaultHome defines kbcli default home name CliDefaultHome = ".kbcli" + + // CliClusterTypeConfigs defines kbcli cluster-type config file name + CliClusterTypeConfigs = "cluster_types" + + // CliChartsCache defines kbcli charts cache dir name + CliChartsCache = "charts" + // CliHomeEnv defines kbcli home system env CliHomeEnv = "KBCLI_HOME" diff --git a/internal/cli/util/helm/downloader.go b/internal/cli/util/helm/downloader.go new file mode 100644 index 00000000000..650ba2a7864 --- /dev/null +++ b/internal/cli/util/helm/downloader.go @@ -0,0 +1,60 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package helm + +import ( + "io" + "strings" + + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/downloader" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/registry" +) + +func NewDownloader(cfg *Config) (*downloader.ChartDownloader, error) { + var err error + var out strings.Builder + + settings := cli.New() + settings.SetNamespace(cfg.namespace) + settings.KubeConfig = cfg.kubeConfig + if cfg.kubeContext != "" { + settings.KubeContext = cfg.kubeContext + } + settings.Debug = cfg.debug + client, err := registry.NewClient( + registry.ClientOptDebug(settings.Debug), + registry.ClientOptEnableCache(true), + registry.ClientOptWriter(io.Discard), + registry.ClientOptCredentialsFile(settings.RegistryConfig), + ) + if err != nil { + return nil, err + } + chartsDownloaders := &downloader.ChartDownloader{ + Out: &out, + Verify: downloader.VerifyNever, + Getters: getter.All(settings), + Options: []getter.Option{}, + RegistryClient: client, + } + return chartsDownloaders, nil +}