add generate images-sync-config command (#472)

* add generate images-sync-config command * use image len * add generate config path * add source registry to config * use backticks instead of double quotes * add spaces
rancher · Aug 28, 2024 · 1e75437 · 1e75437
1 parent 141c6a0
commit 1e75437
Show file tree

Hide file tree

Showing 3 changed files with 176 additions and 12 deletions.
diff --git a/cmd/release/cmd/generate.go b/cmd/release/cmd/generate.go
@@ -35,6 +35,10 @@ var (
 	rancherImagesDigestsOutputFile      string
 	rancherImagesDigestsRegistry        string
 	rancherImagesDigestsImagesURL       string
+	rancherSyncImages                   []string
+	rancherSourceRegistry               string
+	rancherTargetRegistry               string
+	rancherSyncConfigOutputPath         string
 )
 
 // generateCmd represents the generate command
@@ -160,6 +164,14 @@ var rancherGenerateDockerImagesDigestsSubCmd = &cobra.Command{
 	},
 }
 
+var rancherGenerateImagesSyncConfigSubCmd = &cobra.Command{
+	Use:   "images-sync-config",
+	Short: "Generate a regsync config file for images sync",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return rancher.GenerateImagesSyncConfig(rancherSyncImages, rancherSourceRegistry, rancherTargetRegistry, rancherSyncConfigOutputPath)
+	},
+}
+
 func init() {
 	rootCmd.AddCommand(generateCmd)
 
@@ -169,6 +181,7 @@ func init() {
 	rancherGenerateSubCmd.AddCommand(rancherGenerateArtifactsIndexSubCmd)
 	rancherGenerateSubCmd.AddCommand(rancherGenerateMissingImagesListSubCmd)
 	rancherGenerateSubCmd.AddCommand(rancherGenerateDockerImagesDigestsSubCmd)
+	rancherGenerateSubCmd.AddCommand(rancherGenerateImagesSyncConfigSubCmd)
 
 	generateCmd.AddCommand(k3sGenerateSubCmd)
 	generateCmd.AddCommand(rke2GenerateSubCmd)
@@ -230,4 +243,21 @@ func init() {
 		fmt.Println(err.Error())
 		os.Exit(1)
 	}
+	// rancher generate images-sync-config
+	rancherGenerateImagesSyncConfigSubCmd.Flags().StringSliceVarP(&rancherSyncImages, "images", "k", make([]string, 0), "List of images to sync to a registry")
+	rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherSourceRegistry, "source-registry", "s", "", "Source registry, where the images are located")
+	rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherTargetRegistry, "target-registry", "t", "", "Target registry, where the images should be synced to")
+	rancherGenerateImagesSyncConfigSubCmd.Flags().StringVarP(&rancherSyncConfigOutputPath, "output", "o", "./config.yaml", "Output path of the generated config file")
+	if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("images"); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("source-registry"); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	if err := rancherGenerateImagesSyncConfigSubCmd.MarkFlagRequired("target-registry"); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
 }
diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go
@@ -32,6 +32,7 @@ import (
 	"github.com/rancher/ecm-distro-tools/repository"
 	"golang.org/x/mod/semver"
 	"golang.org/x/sync/errgroup"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -53,28 +54,43 @@ const (
 	dockerService                 = "registry.docker.io"
 )
 
+var regsyncDefaultMediaTypes = []string{
+	"application/vnd.docker.distribution.manifest.v2+json",
+	"application/vnd.docker.distribution.manifest.list.v2+json",
+	"application/vnd.oci.image.manifest.v1+json",
+	"application/vnd.oci.image.index.v1+json",
+}
+
 var registriesInfo = map[string]registryInfo{
 	"registry.rancher.com": {
-		BaseURL: rancherRegistryBaseURL,
-		AuthURL: sccSUSEURL,
-		Service: sccSUSEService,
+		BaseURL:     rancherRegistryBaseURL,
+		AuthURL:     sccSUSEURL,
+		Service:     sccSUSEService,
+		UserEnv:     `{{env "PRIME_REGISTRY_USERNAME"}}`,
+		PasswordEnv: `{{env "PRIME_REGISTRY_PASSWORD"}}`,
 	},
 	"stgregistry.suse.com": {
-		BaseURL: stagingRancherRegistryBaseURL,
-		AuthURL: stagingSccSUSEURL,
-		Service: sccSUSEService,
+		BaseURL:     stagingRancherRegistryBaseURL,
+		AuthURL:     stagingSccSUSEURL,
+		Service:     sccSUSEService,
+		UserEnv:     `{{env "STAGING_REGISTRY_USERNAME"}}`,
+		PasswordEnv: `{{env "STAGING_REGISTRY_PASSWORD"}}`,
 	},
 	"docker.io": {
-		BaseURL: dockerRegistryURL,
-		AuthURL: dockerAuthURL,
-		Service: dockerService,
+		BaseURL:     dockerRegistryURL,
+		AuthURL:     dockerAuthURL,
+		Service:     dockerService,
+		UserEnv:     `{{env "DOCKERIO_REGISTRY_USERNAME"}}`,
+		PasswordEnv: `{{env "DOCKERIO_REGISTRY_PASSWORD"}}`,
 	},
 }
 
 type registryInfo struct {
-	BaseURL string
-	AuthURL string
-	Service string
+	BaseURL     string
+	AuthURL     string
+	Service     string
+	UserEnv     string
+	PasswordEnv string
 }
 
 type imageDigest map[string]string
@@ -108,6 +124,35 @@ type registryAuthToken struct {
 	Token string `json:"token"`
 }
 
+type regsyncConfig struct {
+	Version  int             `yaml:"version"`
+	Creds    []regsyncCreds  `yaml:"creds"`
+	Defaults regsyncDefaults `yaml:"defaults"`
+	Sync     []regsyncSync   `yaml:"sync"`
+}
+
+type regsyncCreds struct {
+	Registry string `yaml:"registry"`
+	User     string `yaml:"user"`
+	Pass     string `yaml:"pass"`
+}
+
+type regsyncDefaults struct {
+	Parallel   int      `yaml:"parallel"`
+	MediaTypes []string `yaml:"mediaTypes"`
+}
+
+type regsyncTags struct {
+	Allow []string `yaml:"allow"`
+}
+
+type regsyncSync struct {
+	Source string      `yaml:"source"`
+	Target string      `yaml:"target"`
+	Type   string      `yaml:"type"`
+	Tags   regsyncTags `yaml:"tags"`
+}
+
 func listS3Objects(ctx context.Context, s3Client *s3.Client, bucketName string, prefix string) ([]string, error) {
 	var keys []string
 	var continuationToken *string
@@ -497,6 +542,67 @@ func GenerateMissingImagesList(imagesListURL, registry string, concurrencyLimit
 	return missingImages, nil
 }
 
+func GenerateImagesSyncConfig(images []string, sourceRegistry, targetRegistry, outputPath string) error {
+	config, err := generateRegsyncConfig(images, sourceRegistry, targetRegistry)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(outputPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return yaml.NewEncoder(f).Encode(config)
+}
+
+func generateRegsyncConfig(images []string, sourceRegistry, targetRegistry string) (*regsyncConfig, error) {
+	sourceRegistryInfo, ok := registriesInfo[sourceRegistry]
+	if !ok {
+		return nil, errors.New("invalid source registry")
+	}
+	targetRegistryInfo, ok := registriesInfo[targetRegistry]
+	if !ok {
+		return nil, errors.New("invalid target registry")
+	}
+
+	config := regsyncConfig{
+		Version: 1,
+		Creds: []regsyncCreds{
+			{
+				Registry: sourceRegistry,
+				User:     sourceRegistryInfo.UserEnv,
+				Pass:     sourceRegistryInfo.PasswordEnv,
+			},
+			{
+				Registry: targetRegistry,
+				User:     targetRegistryInfo.UserEnv,
+				Pass:     targetRegistryInfo.PasswordEnv,
+			},
+		},
+		Defaults: regsyncDefaults{
+			Parallel:   1,
+			MediaTypes: regsyncDefaultMediaTypes,
+		},
+		Sync: make([]regsyncSync, len(images)),
+	}
+
+	for i, imageAndVersion := range images {
+		image, imageVersion, err := splitImageAndVersion(imageAndVersion)
+		if err != nil {
+			return nil, err
+		}
+		config.Sync[i] = regsyncSync{
+			Source: sourceRegistry + "/" + image,
+			Target: targetRegistry + "/" + image,
+			Type:   "repository",
+			Tags:   regsyncTags{Allow: []string{imageVersion}},
+		}
+	}
+	return &config, nil
+}
+
 func imageSliceToMap(images []string) (map[string]bool, error) {
 	imagesMap := make(map[string]bool, len(images))
 	for _, image := range images {

diff --git a/release/rancher/rancher_test.go b/release/rancher/rancher_test.go
@@ -58,3 +58,31 @@ func TestSplitImageAndVersion(t *testing.T) {
 		t.Error("expected to flag image without version as malformed " + imagesWithoutVersion[0])
 	}
 }
+
+func TestGenerateRegsyncConfig(t *testing.T) {
+	rancherImage := "rancher/rancher"
+	rancherAgentImage := "rancher/rancher-agent"
+	rancherVersion := "v2.9.0"
+	images := []string{rancherImage + ":" + rancherVersion, rancherAgentImage + ":" + rancherVersion}
+	sourceRegistry := "docker.io"
+	targetRegistry := "registry.rancher.com"
+	sourceRancherImage := sourceRegistry + "/" + rancherImage
+	sourceRancherAgentImage := sourceRegistry + "/" + rancherAgentImage
+	targetRancherImage := targetRegistry + "/" + rancherImage
+	config, err := generateRegsyncConfig(images, sourceRegistry, targetRegistry)
+	if err != nil {
+		t.Error(err)
+	}
+	if config.Sync[0].Source != sourceRancherImage {
+		t.Error("rancher image should be: '" + sourceRancherImage + "' instead, got: '" + config.Sync[0].Source + "'")
+	}
+	if config.Sync[0].Target != targetRancherImage {
+		t.Error("target rancher image should be: '" + targetRancherImage + "' instead, got: '" + config.Sync[0].Target + "'")
+	}
+	if config.Sync[0].Tags.Allow[0] != rancherVersion {
+		t.Error("rancher version should be: '" + rancherVersion + "' instead, got: '" + config.Sync[0].Tags.Allow[0] + "'")
+	}
+	if config.Sync[1].Source != sourceRancherAgentImage {
+		t.Error("rancher agent image should be: '" + sourceRancherAgentImage + "' instead, got: '" + config.Sync[1].Source + "'")
+	}
+}