From b8d8cf3501cf38d68f115c3e1eb043934c0882f2 Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Tue, 23 Jul 2024 19:04:51 +0900 Subject: [PATCH 1/6] feat: migrate InvokeGenerationFromSheet Signed-off-by: dusdjhyeon --- generators/registry.go | 331 +++++++++++++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 2 + 3 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 generators/registry.go diff --git a/generators/registry.go b/generators/registry.go new file mode 100644 index 00000000..470989f3 --- /dev/null +++ b/generators/registry.go @@ -0,0 +1,331 @@ +package generators + +import ( + "context" + "fmt" + "os" + "strconv" + "path/filepath" + "encoding/json" + "net/url" + "sync" + "time" + + "google.golang.org/api/sheets/v4" + "github.com/layer5io/meshkit/utils/walker" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/generators/github" + "github.com/layer5io/meshkit/utils/store" + "github.com/layer5io/meshkit/utils/registry" + "golang.org/x/sync/semaphore" + "github.com/layer5io/meshkit/logger" +) + +var ( + srv *sheets.Service + totalAggregateModel int + totalAggregateComponents int + componentSpredsheetGID int64 + sheetGID int64 + registryLocation string + logFile *os.File + errorLogFile *os.File + Log logger.Handler + LogError logger.Handler +) + +var ( + artifactHubCount = 0 + artifactHubRateLimit = 100 + artifactHubRateLimitDur = 5 * time.Minute + artifactHubMutex sync.Mutex + defVersion = "v1.0.0" + GoogleSpreadSheetURL = "https://docs.google.com/spreadsheets/d/" + logDirPath = filepath.Join(utils.GetHome(), ".meshery", "logs", "registry") + modelToCompGenerateTracker = store.NewGenericThreadSafeStore[registry.CompGenerateTracker]() +) + +func InvokeGenerationFromSheet(wg *sync.WaitGroup, spreadsheetID string, spreadsheetCred string) error { + + weightedSem := semaphore.NewWeighted(20) + url := GoogleSpreadSheetURL + spreadsheetID + totalAvailableModels := 0 + spreadsheeetChan := make(chan registry.SpreadsheetData) + + defer func() { + logModelGenerationSummary(modelToCompGenerateTracker) + + Log.UpdateLogOutput(os.Stdout) + Log.UpdateLogOutput(os.Stdout) + Log.Info(fmt.Sprintf("Summary: %d models, %d components generated.", totalAggregateModel, totalAggregateComponents)) + + Log.Info("See ", logDirPath, " for detailed logs.") + + _ = logFile.Close() + _ = errorLogFile.Close() + totalAggregateModel = 0 + totalAggregateComponents = 0 + }() + + modelCSVHelper, err := parseModelSheet(url) + if err != nil { + return err + } + + componentCSVHelper, err := parseComponentSheet(url) + if err != nil { + return err + } + + Log.UpdateLogOutput(logFile) + Log.UpdateLogOutput(errorLogFile) + var wgForSpreadsheetUpdate sync.WaitGroup + wgForSpreadsheetUpdate.Add(1) + go func() { + registry.ProcessModelToComponentsMap(componentCSVHelper.Components) + registry.VerifyandUpdateSpreadsheet(spreadsheetCred, &wgForSpreadsheetUpdate, srv, spreadsheeetChan, spreadsheetID) + }() + + // Iterate models from the spreadsheet + for _, model := range modelCSVHelper.Models { + totalAvailableModels++ + + ctx := context.Background() + + err := weightedSem.Acquire(ctx, 1) + if err != nil { + break + } + + wg.Add(1) + go func(model registry.ModelCSV) { + defer func() { + wg.Done() + weightedSem.Release(1) + }() + if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "meshery" { + err = GenerateDefsForCoreRegistrant(model) + if err != nil { + LogError.Error(err) + } + return + } + + generator, err := NewGenerator(model.Registrant, model.SourceURL, model.Model) + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + + if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "artifacthub" { + rateLimitArtifactHub() + + } + pkg, err := generator.GetPackage() + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + + version := pkg.GetVersion() + modelDirPath, compDirPath, err := CreateVersionedDirectoryForModelAndComp(version, model.Model) + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + modelDef, err := writeModelDefToFileSystem(&model, version, modelDirPath) + if err != nil { + LogError.Error(err) + return + } + + comps, err := pkg.GenerateComponents() + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + Log.Info("Current model: ", model.Model) + Log.Info(" extracted ", len(comps), " components for ", model.ModelDisplayName, " (", model.Model, ")") + for _, comp := range comps { + comp.Version = defVersion + if comp.Metadata == nil { + comp.Metadata = make(map[string]interface{}) + } + // Assign the component status corresponding to model status. + // i.e. If model is enabled comps are also "enabled". Ultimately all individual comps itself will have ability to control their status. + // The status "enabled" indicates that the component will be registered inside the registry. + comp.Model = *modelDef + assignDefaultsForCompDefs(&comp, modelDef) + err := comp.WriteComponentDefinition(compDirPath) + if err != nil { + Log.Info(err) + } + } + + spreadsheeetChan <- registry.SpreadsheetData{ + Model: &model, + Components: comps, + } + + modelToCompGenerateTracker.Set(model.Model, registry.CompGenerateTracker{ + TotalComps: len(comps), + Version: version, + }) + }(model) + + } + wg.Wait() + close(spreadsheeetChan) + wgForSpreadsheetUpdate.Wait() + return nil +} + +func writeModelDefToFileSystem(model *registry.ModelCSV, version, modelDefPath string) (*v1beta1.Model, error) { + modelDef := model.CreateModelDefinition(version, defVersion) + err := modelDef.WriteModelDefinition(modelDefPath+"/model.json", "json") + if err != nil { + return nil, err + } + + return &modelDef, nil +} + +func rateLimitArtifactHub() { + artifactHubMutex.Lock() + defer artifactHubMutex.Unlock() + + if artifactHubCount > 0 && artifactHubCount%artifactHubRateLimit == 0 { + Log.Info("Rate limit reached for Artifact Hub. Sleeping for 5 minutes...") + time.Sleep(artifactHubRateLimitDur) + } + artifactHubCount++ +} + +func logModelGenerationSummary(modelToCompGenerateTracker *store.GenerticThreadSafeStore[registry.CompGenerateTracker]) { + for key, val := range modelToCompGenerateTracker.GetAllPairs() { + Log.Info(fmt.Sprintf("Generated %d components for model [%s] %s", val.TotalComps, key, val.Version)) + totalAggregateComponents += val.TotalComps + totalAggregateModel++ + } + + Log.Info(fmt.Sprintf("-----------------------------\n-----------------------------\nGenerated %d models and %d components", totalAggregateModel, totalAggregateComponents)) +} + +func parseModelSheet(url string) (*registry.ModelCSVHelper, error) { + modelCSVHelper, err := registry.NewModelCSVHelper(url, "Models", sheetGID) + if err != nil { + return nil, err + } + + err = modelCSVHelper.ParseModelsSheet(false) + if err != nil { + return nil, registry.ErrGenerateModel(err, "unable to start model generation") + } + return modelCSVHelper, nil +} + +func parseComponentSheet(url string) (*registry.ComponentCSVHelper, error) { + compCSVHelper, err := registry.NewComponentCSVHelper(url, "Components", componentSpredsheetGID) + if err != nil { + return nil, err + } + err = compCSVHelper.ParseComponentsSheet() + if err != nil { + return nil, registry.ErrGenerateModel(err, "unable to start model generation") + } + return compCSVHelper, nil +} + +func CreateVersionedDirectoryForModelAndComp(version, modelName string) (string, string, error) { + modelDirPath := filepath.Join(registryLocation, modelName, version, defVersion) + err := utils.CreateDirectory(modelDirPath) + if err != nil { + return "", "", err + } + + compDirPath := filepath.Join(modelDirPath, "components") + err = utils.CreateDirectory(compDirPath) + return modelDirPath, compDirPath, err +} + + +func GenerateDefsForCoreRegistrant(model registry.ModelCSV) error { + totalComps := 0 + var version string + defer func() { + modelToCompGenerateTracker.Set(model.Model, registry.CompGenerateTracker{ + TotalComps: totalComps, + Version: version, + }) + }() + + path, err := url.Parse(model.SourceURL) + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + LogError.Error(err) + return nil + } + gitRepo := github.GitRepo{ + URL: path, + PackageName: model.Model, + } + owner, repo, branch, root, err := gitRepo.ExtractRepoDetailsFromSourceURL() + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + LogError.Error(err) + return nil + } + + isModelPublished, _ := strconv.ParseBool(model.PublishToRegistry) + //Initialize walker + gitWalker := walker.NewGit() + if isModelPublished { + gw := gitWalker. + Owner(owner). + Repo(repo). + Branch(branch). + Root(root). + RegisterFileInterceptor(func(f walker.File) error { + // Check if the file has a JSON extension + if filepath.Ext(f.Name) != ".json" { + return nil + } + contentBytes := []byte(f.Content) + var componentDef v1beta1.ComponentDefinition + if err := json.Unmarshal(contentBytes, &componentDef); err != nil { + return err + } + version = componentDef.Model.Model.Version + modelDirPath, compDirPath, err := CreateVersionedDirectoryForModelAndComp(version, model.Model) + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + return err + } + _, err = writeModelDefToFileSystem(&model, version, modelDirPath) // how to infer this? @Beginner86 any idea? new column? + if err != nil { + return registry.ErrGenerateModel(err, model.Model) + } + + err = componentDef.WriteComponentDefinition(compDirPath) + if err != nil { + err = registry.ErrGenerateComponent(err, model.Model, componentDef.DisplayName) + LogError.Error(err) + } + return nil + }) + err = gw.Walk() + if err != nil { + return err + } + } + + return nil +} + +func assignDefaultsForCompDefs(componentDef *v1beta1.ComponentDefinition, modelDef *v1beta1.Model) { + componentDef.Metadata["status"] = modelDef.Status + for k, v := range modelDef.Metadata { + componentDef.Metadata[k] = v + } +} diff --git a/go.mod b/go.mod index d413cafa..4d7f0f6f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/fluxcd/pkg/tar v0.4.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-logr/logr v1.3.0 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/gofrs/uuid v4.4.0+incompatible github.com/google/go-containerregistry v0.17.0 github.com/google/uuid v1.5.0 @@ -33,6 +34,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.17.0 golang.org/x/oauth2 v0.15.0 + golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 google.golang.org/api v0.152.0 gopkg.in/yaml.v2 v2.4.0 @@ -237,7 +239,6 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index d2feebea..479ffbba 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,8 @@ github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XE github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= From 7b7290ffd67af80e4fcf45023a9001da7614c640 Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Tue, 23 Jul 2024 19:05:17 +0900 Subject: [PATCH 2/6] feat: migrate meshery utils for InvokeGenerationFromSheet Signed-off-by: dusdjhyeon --- utils/registry/component.go | 321 +++++++++++++++++++++++++++ utils/registry/error.go | 51 +++++ utils/registry/helper.go | 181 +++++++++++++++ utils/registry/spreadsheet_update.go | 224 +++++++++++++++++++ utils/registry/types.go | 61 +++++ 5 files changed, 838 insertions(+) create mode 100644 utils/registry/component.go create mode 100644 utils/registry/error.go create mode 100644 utils/registry/helper.go create mode 100644 utils/registry/spreadsheet_update.go create mode 100644 utils/registry/types.go diff --git a/utils/registry/component.go b/utils/registry/component.go new file mode 100644 index 00000000..9869c1ae --- /dev/null +++ b/utils/registry/component.go @@ -0,0 +1,321 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/logger" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/csv" + "github.com/layer5io/meshkit/utils/manifests" +) + +const ( + SVG_WIDTH = 20 + SVG_HEIGHT = 20 +) + +var ( + Log logger.Handler + LogError logger.Handler +) + +type ComponentCSV struct { + Registrant string `json:"registrant" csv:"registrant"` + Model string `json:"model" csv:"model"` + Component string `json:"component" csv:"component"` + Description string `json:"description" csv:"description"` + Shape string `json:"shape" csv:"shape"` + PrimaryColor string `json:"primaryColor" csv:"primaryColor"` + SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` + SVGColor string `json:"svgColor" csv:"svgColor"` + SVGWhite string `json:"svgWhite" csv:"svgWhite"` + SVGComplete string `json:"svgComplete" csv:"svgComplete"` + HasSchema string `json:"hasSchema" csv:"hasSchema"` + Docs string `json:"docs" csv:"docs"` + StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` + Styles string `json:"styles" csv:"styles"` + ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` + DefaultData string `json:"defaultData" csv:"defaultData"` + Capabilities string `json:"capabilities" csv:"capabilities"` + LogoURL string `json:"logoURL" csv:"logoURL"` + Genealogy string `json:"genealogy" csv:"genealogy"` + IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` + + ModelDisplayName string `json:"modelDisplayName" csv:"-"` + Category string `json:"category" csv:"-"` + SubCategory string `json:"subCategory" csv:"-"` +} + +// The Component Definition generated assumes or is only for components which have registrant as "meshery" +func (c *ComponentCSV) CreateComponentDefinition(isModelPublished bool, defVersion string) (v1beta1.ComponentDefinition, error) { + componentDefinition := &v1beta1.ComponentDefinition{ + VersionMeta: v1beta1.VersionMeta{ + SchemaVersion: v1beta1.ComponentSchemaVersion, + Version: defVersion, + }, + DisplayName: c.Component, + Format: "JSON", + Metadata: map[string]interface{}{ + "published": isModelPublished, + }, + Component: v1beta1.ComponentEntity{}, + } + err := c.UpdateCompDefinition(componentDefinition) + return *componentDefinition, err +} + +var compMetadataValues = []string{ + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "genealogy", "isAnnotation", "shape", "subCategory", +} + +func (c *ComponentCSV) UpdateCompDefinition(compDef *v1beta1.ComponentDefinition) error { + + metadata := map[string]interface{}{} + compMetadata, err := utils.MarshalAndUnmarshal[ComponentCSV, map[string]interface{}](*c) + if err != nil { + return err + } + metadata = utils.MergeMaps(metadata, compDef.Metadata) + + for _, key := range compMetadataValues { + if key == "svgColor" || key == "svgWhite" { + svg, err := utils.Cast[string](compMetadata[key]) + if err == nil { + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) + if err != nil { + // If svg cannot be updated, assign the svg value as it is + metadata[key] = compMetadata[key] + } + } + } + metadata[key] = compMetadata[key] + } + + isAnnotation := false + if strings.ToLower(c.IsAnnotation) == "true" { + isAnnotation = true + } + metadata["isAnnotation"] = isAnnotation + compDef.Metadata = metadata + return nil +} + +type ComponentCSVHelper struct { + SpreadsheetID int64 + SpreadsheetURL string + Title string + CSVPath string + Components map[string]map[string][]ComponentCSV +} + +func NewComponentCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ComponentCSVHelper, error) { + sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) + Log.Info("Downloading CSV from: ", sheetURL) + dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") + _ = os.MkdirAll(dirPath, 0755) + csvPath := filepath.Join(dirPath, "components.csv") + err := utils.DownloadFile(csvPath, sheetURL) + if err != nil { + return nil, utils.ErrReadingRemoteFile(err) + } + + return &ComponentCSVHelper{ + SpreadsheetID: spreadsheetID, + SpreadsheetURL: sheetURL, + Title: spreadsheetName, + CSVPath: csvPath, + Components: make(map[string]map[string][]ComponentCSV), + }, nil +} + +func (mch *ComponentCSVHelper) GetColumns() ([]string, error) { + csvReader, err := csv.NewCSVParser[ComponentCSV](mch.CSVPath, rowIndex, nil, func(_ []string, _ []string) bool { + return true + }) + if err != nil { + return nil, err + } + + return csvReader.ExtractCols(rowIndex) +} + +func (mch *ComponentCSVHelper) ParseComponentsSheet() error { + ch := make(chan ComponentCSV, 1) + errorChan := make(chan error, 1) + csvReader, err := csv.NewCSVParser[ComponentCSV](mch.CSVPath, rowIndex, nil, func(_ []string, _ []string) bool { + return true + }) + + if err != nil { + return ErrFileRead(err) + } + + go func() { + Log.Info("Parsing Components...") + + err := csvReader.Parse(ch, errorChan) + if err != nil { + errorChan <- err + } + }() + + for { + select { + + case data := <-ch: + if mch.Components[data.Registrant] == nil { + mch.Components[data.Registrant] = make(map[string][]ComponentCSV, 0) + } + if mch.Components[data.Registrant][data.Model] == nil { + mch.Components[data.Registrant][data.Model] = make([]ComponentCSV, 0) + } + mch.Components[data.Registrant][data.Model] = append(mch.Components[data.Registrant][data.Model], data) + Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s] component [%s]", data.Registrant, data.Model, data.Component)) + case err := <-errorChan: + Log.Error(err) + + case <-csvReader.Context.Done(): + return nil + } + } +} + +func CreateComponentsMetadataAndCreateSVGsForMDXStyle(model ModelCSV, components []ComponentCSV, path, svgDir string) (string, error) { + err := os.MkdirAll(filepath.Join(path, svgDir), 0777) + if err != nil { + return "", err + } + componentMetadata := `[` + for idx, comp := range components { + componentTemplate := ` +{ +"name": "%s", +"colorIcon": "%s", +"whiteIcon": "%s", +"description": "%s", +}` + + // add comma if not last component + if idx != len(components)-1 { + componentTemplate += "," + } + + compName := utils.FormatName(manifests.FormatToReadableString(comp.Component)) + colorIconDir := filepath.Join(svgDir, compName, "icons", "color") + whiteIconDir := filepath.Join(svgDir, compName, "icons", "white") + + componentMetadata += fmt.Sprintf(componentTemplate, compName, fmt.Sprintf("%s/%s-color.svg", colorIconDir, compName), fmt.Sprintf("%s/%s-white.svg", whiteIconDir, compName), comp.Description) + + // create color svg dir + err = os.MkdirAll(filepath.Join(path, colorIconDir), 0777) + if err != nil { + return "", err + } + + // create white svg dir + err = os.MkdirAll(filepath.Join(path, whiteIconDir), 0777) + if err != nil { + return "", err + } + + colorSVG, whiteSVG := getSVGForComponent(model, comp) + err = utils.WriteToFile(filepath.Join(path, colorIconDir, compName+"-color.svg"), colorSVG) + if err != nil { + return "", err + } + err = utils.WriteToFile(filepath.Join(path, whiteIconDir, compName+"-white.svg"), whiteSVG) + if err != nil { + return "", err + } + } + + componentMetadata += `]` + + return componentMetadata, nil +} + +func CreateComponentsMetadataAndCreateSVGsForMDStyle(model ModelCSV, components []ComponentCSV, path, svgDir string) (string, error) { + err := os.MkdirAll(filepath.Join(path), 0777) + if err != nil { + return "", err + } + componentMetadata := "" + for _, comp := range components { + componentTemplate := ` +- name: %s + colorIcon: %s + whiteIcon: %s + description: %s` + + compName := utils.FormatName(manifests.FormatToReadableString(comp.Component)) + colorIconDir := filepath.Join(svgDir, compName, "icons", "color") + whiteIconDir := filepath.Join(svgDir, compName, "icons", "white") + + componentMetadata += fmt.Sprintf(componentTemplate, compName, fmt.Sprintf("%s/%s-color.svg", colorIconDir, compName), fmt.Sprintf("%s/%s-white.svg", whiteIconDir, compName), comp.Description) + + // create color svg dir + err = os.MkdirAll(filepath.Join(path, compName, "icons", "color"), 0777) + if err != nil { + return "", err + } + + // create white svg dir + err = os.MkdirAll(filepath.Join(path, compName, "icons", "white"), 0777) + if err != nil { + return "", err + } + + colorSVG, whiteSVG := getSVGForComponent(model, comp) + err = utils.WriteToFile(filepath.Join(path, compName, "icons", "color", compName+"-color.svg"), colorSVG) + if err != nil { + return "", err + } + err = utils.WriteToFile(filepath.Join(path, compName, "icons", "white", compName+"-white.svg"), whiteSVG) + if err != nil { + return "", err + } + } + + return componentMetadata, nil +} + +func (m ComponentCSVHelper) Cleanup() error { + // remove csv file + Log.Info("Removing CSV file: ", m.CSVPath) + err := os.Remove(m.CSVPath) + if err != nil { + return err + } + return nil +} + +func ConvertCompDefToCompCSV(modelcsv *ModelCSV, compDef v1beta1.ComponentDefinition) *ComponentCSV { + compCSV, _ := utils.MarshalAndUnmarshal[map[string]interface{}, ComponentCSV](compDef.Metadata) + compCSV.Registrant = modelcsv.Registrant + compCSV.Model = modelcsv.Model + compCSV.Component = compDef.Component.Kind + compCSV.ModelDisplayName = modelcsv.ModelDisplayName + compCSV.Category = modelcsv.Category + compCSV.SubCategory = modelcsv.SubCategory + + return &compCSV +} + +func getSVGForComponent(model ModelCSV, component ComponentCSV) (colorSVG string, whiteSVG string) { + colorSVG = component.SVGColor + whiteSVG = component.SVGWhite + + if colorSVG == "" { + colorSVG = model.SVGColor + } + + if whiteSVG == "" { + whiteSVG = model.SVGWhite + } + return +} diff --git a/utils/registry/error.go b/utils/registry/error.go new file mode 100644 index 00000000..890b5987 --- /dev/null +++ b/utils/registry/error.go @@ -0,0 +1,51 @@ +package registry + +import ( + "fmt" + "github.com/layer5io/meshkit/errors" +) + +var ( + ErrAppendToSheetCode = "mesheryctl-1116" + ErrCreateDirCode = "mesheryctl-1066" + ErrGenerateComponentCode = "mesheryctl-1056" + ErrGenerateModelCode = "mesheryctl-1055" + ErrFileReadCode = "mesheryctl-1095" + ErrMarshalStructToCSVCode = "mesheryctl-1115" +) + +func ErrGenerateModel(err error, modelName string) error { + return errors.New(ErrGenerateModelCode, errors.Alert, []string{fmt.Sprintf("error generating model: %s", modelName)}, []string{fmt.Sprintf("Error generating model: %s\n %s", modelName, err.Error())}, []string{"Registrant used for the model is not supported", "Verify the model's source URL.", "Failed to create a local directory in the filesystem for this model."}, []string{"Ensure that each kind of registrant used is a supported kind.", "Ensure correct model source URL is provided and properly formatted.", "Ensure sufficient permissions to allow creation of model directory."}) +} + +func ErrCreateDir(err error, obj string) error { + return errors.New(ErrCreateDirCode, errors.Alert, []string{"Error creating directory ", obj}, []string{err.Error()}, []string{}, []string{}) +} + +func ErrGenerateComponent(err error, modelName, compName string) error { + return errors.New(ErrGenerateComponentCode, errors.Alert, []string{"error generating comp %s of model %s", compName, modelName}, []string{err.Error()}, []string{}, []string{}) +} + +func ErrFileRead(err error) error { + return errors.New(ErrFileReadCode, errors.Alert, + []string{"File read error"}, + []string{err.Error()}, + []string{"The provided file is not present or has an invalid path."}, + []string{"To proceed, provide a valid file path with a valid file."}) +} + +func ErrAppendToSheet(err error, id string) error { + return errors.New(ErrAppendToSheetCode, errors.Alert, + []string{fmt.Sprintf("Failed to append data into sheet %s", id)}, + []string{err.Error()}, + []string{"Error occurred while appending to the spreadsheet", "The credential might be incorrect/expired"}, + []string{"Ensure correct append range (A1 notation) is used", "Ensure correct credential is used"}) +} + +func ErrMarshalStructToCSV(err error) error { + return errors.New(ErrMarshalStructToCSVCode, errors.Alert, + []string{"Failed to marshal struct to csv"}, + []string{err.Error()}, + []string{"The column names in your spreadsheet do not match the names in the struct.", " For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}, + []string{"The column names in the spreadsheet do not match the names in the struct. Please make sure they are spelled exactly the same and use the same case (uppercase/lowercase).", "The value you are trying to convert is not of the expected type for the column. Please ensure it is a [number, string, date, etc.].", "The column names in your spreadsheet do not match the names in the struct. For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}) +} \ No newline at end of file diff --git a/utils/registry/helper.go b/utils/registry/helper.go new file mode 100644 index 00000000..15770076 --- /dev/null +++ b/utils/registry/helper.go @@ -0,0 +1,181 @@ +package registry + +import ( + "fmt" + "os" + "strconv" + "strings" + "path/filepath" + "google.golang.org/api/sheets/v4" + "github.com/layer5io/meshkit/utils/store" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/csv" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" +) + +var ( + rowIndex = 1 + shouldRegisterColIndex = -1 + shouldRegisterMod = "publishToSites" + modelToCompGenerateTracker = store.NewGenericThreadSafeStore[CompGenerateTracker]() +) + +var modelMetadataValues = []string{ + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "isAnnotation", "shape", +} + +func NewModelCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ModelCSVHelper, error) { + sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) + Log.Info("Downloading CSV from: ", sheetURL) + dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return nil, ErrCreateDir(err, dirPath) + } + csvPath := filepath.Join(dirPath, "models.csv") + err = utils.DownloadFile(csvPath, sheetURL) + if err != nil { + return nil, utils.ErrReadingRemoteFile(err) + } + + return &ModelCSVHelper{ + SpreadsheetID: spreadsheetID, + SpreadsheetURL: sheetURL, + Models: []ModelCSV{}, + CSVPath: csvPath, + Title: spreadsheetName, + }, nil +} + +func GetSheetIDFromTitle(s *sheets.Spreadsheet, title string) int64 { + for _, sheet := range s.Sheets { + if sheet.Properties.Title == title { + return sheet.Properties.SheetId + } + } + return -1 +} + +func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { + ch := make(chan ModelCSV, 1) + errorChan := make(chan error, 1) + csvReader, err := csv.NewCSVParser[ModelCSV](mch.CSVPath, rowIndex, nil, func(columns []string, currentRow []string) bool { + index := 0 + + if parseForDocs { + index = GetIndexForRegisterCol(columns, shouldRegisterMod) + } else { + // Generation of models should not consider publishedToRegistry column value. + // Generation should happen for all models, while during registration "published" attribute should be respected. + return true + } + if index != -1 && index < len(currentRow) { + shouldRegister := currentRow[index] + return strings.ToLower(shouldRegister) == "true" + } + return false + }) + + if err != nil { + return ErrFileRead(err) + } + + go func() { + Log.Info("Parsing Models...") + err := csvReader.Parse(ch, errorChan) + if err != nil { + errorChan <- err + } + }() + for { + select { + + case data := <-ch: + mch.Models = append(mch.Models, data) + Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s]", data.Registrant, data.Model)) + case err := <-errorChan: + return ErrFileRead(err) + + case <-csvReader.Context.Done(): + return nil + } + } +} + +func (mcv *ModelCSV) CreateModelDefinition(version, defVersion string) v1beta1.Model { + status := entity.Ignored + if strings.ToLower(mcv.PublishToRegistry) == "true" { + status = entity.Enabled + } + + model := v1beta1.Model{ + VersionMeta: v1beta1.VersionMeta{ + Version: defVersion, + SchemaVersion: v1beta1.ModelSchemaVersion, + }, + Name: mcv.Model, + DisplayName: mcv.ModelDisplayName, + Status: status, + Registrant: v1beta1.Host{ + Hostname: utils.ReplaceSpacesAndConvertToLowercase(mcv.Registrant), + }, + Category: v1beta1.Category{ + Name: mcv.Category, + }, + SubCategory: mcv.SubCategory, + Model: v1beta1.ModelEntity{ + Version: version, + }, + } + err := mcv.UpdateModelDefinition(&model) + if err != nil { + Log.Error(err) + } + return model +} + +func (m *ModelCSV) UpdateModelDefinition(modelDef *v1beta1.Model) error { + + metadata := map[string]interface{}{} + modelMetadata, err := utils.MarshalAndUnmarshal[ModelCSV, map[string]interface{}](*m) + if err != nil { + return err + } + metadata = utils.MergeMaps(metadata, modelDef.Metadata) + + for _, key := range modelMetadataValues { + if key == "svgColor" || key == "svgWhite" { + svg, err := utils.Cast[string](modelMetadata[key]) + if err == nil { + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) + if err != nil { + // If svg cannot be updated, assign the svg value as it is + metadata[key] = modelMetadata[key] + } + } + } + metadata[key] = modelMetadata[key] + } + + isAnnotation := false + if strings.ToLower(m.IsAnnotation) == "true" { + isAnnotation = true + } + metadata["isAnnotation"] = isAnnotation + modelDef.Metadata = metadata + return nil +} + +func GetIndexForRegisterCol(cols []string, shouldRegister string) int { + if shouldRegisterColIndex != -1 { + return shouldRegisterColIndex + } + + for index, col := range cols { + if col == shouldRegister { + return index + } + } + return shouldRegisterColIndex +} \ No newline at end of file diff --git a/utils/registry/spreadsheet_update.go b/utils/registry/spreadsheet_update.go new file mode 100644 index 00000000..b1f5716a --- /dev/null +++ b/utils/registry/spreadsheet_update.go @@ -0,0 +1,224 @@ +package registry + +import ( + "bytes" + "context" + "sync" + + cuecsv "cuelang.org/go/pkg/encoding/csv" + "github.com/gocarina/gocsv" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "google.golang.org/api/sheets/v4" +) + +var ( + compBatchSize = 100 + modelBatchSize = 100 + + // refers to all cells in the fourth row (explain how is table clulte in ghsset and henc this rang is valid and will not reuire udpate or when it hould require update) + ComponentsSheetAppendRange = "Components!A4" + ModelsSheetAppendRange = "Models!A4" +) + +// registrant:model:component:[true/false] +// Tracks if component sheet requires update +var RegistrantToModelsToComponentsMap = make(map[string]map[string]map[string]bool) + +// registrant:model:[true/false] +// Tracks if component sheet requires update +var RegistrantToModelsMap = make(map[string]map[string]bool) + +func ProcessModelToComponentsMap(existingComponents map[string]map[string][]ComponentCSV) { + RegistrantToModelsToComponentsMap = make(map[string]map[string]map[string]bool, len(existingComponents)) + for registrant, models := range existingComponents { + for model, comps := range models { + if RegistrantToModelsToComponentsMap[registrant] == nil { + RegistrantToModelsToComponentsMap[registrant] = make(map[string]map[string]bool) + } + for _, comp := range comps { + if RegistrantToModelsToComponentsMap[registrant][model] == nil { + RegistrantToModelsToComponentsMap[registrant][model] = make(map[string]bool) + } + RegistrantToModelsToComponentsMap[registrant][model][comp.Component] = true + } + } + } +} + +func addEntriesInCompUpdateList(modelEntry *ModelCSV, compEntries []v1beta1.ComponentDefinition, compList []*ComponentCSV) []*ComponentCSV { + registrant := modelEntry.Registrant + model := modelEntry.Model + + if RegistrantToModelsToComponentsMap[registrant][model] == nil { + RegistrantToModelsToComponentsMap[registrant][model] = make(map[string]bool) + } + + for _, comp := range compEntries { + if !RegistrantToModelsToComponentsMap[registrant][model][comp.Component.Kind] { + RegistrantToModelsToComponentsMap[registrant][model][comp.Component.Kind] = true + compList = append(compList, ConvertCompDefToCompCSV(modelEntry, comp)) + compBatchSize-- + } + } + + return compList +} + +func addEntriesInModelUpdateList(modelEntry *ModelCSV, modelList []*ModelCSV) []*ModelCSV { + registrant := modelEntry.Registrant + + if RegistrantToModelsMap[registrant] == nil { + RegistrantToModelsMap[registrant] = make(map[string]bool) + } + RegistrantToModelsMap[registrant][modelEntry.Model] = true + modelBatchSize-- + + return modelList +} + +// Verifies if the component entry already exist in the spreadsheet, otherwise updates the spreadshhet to include new component entry. +func VerifyandUpdateSpreadsheet(cred string, wg *sync.WaitGroup, srv *sheets.Service, spreadsheetUpdateChan chan SpreadsheetData, sheetId string) { + defer wg.Done() + + entriesToBeAddedInCompSheet := []*ComponentCSV{} + entriesToBeAddedInModelSheet := []*ModelCSV{} + + for data := range spreadsheetUpdateChan { + _, ok := RegistrantToModelsMap[data.Model.Registrant] + if !ok { + entriesToBeAddedInModelSheet = addEntriesInModelUpdateList(data.Model, entriesToBeAddedInModelSheet) + } + + for _, comp := range data.Components { + existingModels, ok := RegistrantToModelsToComponentsMap[data.Model.Registrant] // replace with registrantr + if ok { + + existingComps, ok := existingModels[data.Model.Model] + + if ok { + entryExist := existingComps[comp.Component.Kind] + + if !entryExist { + entriesToBeAddedInCompSheet = append(entriesToBeAddedInCompSheet, ConvertCompDefToCompCSV(data.Model, comp)) + compBatchSize-- + RegistrantToModelsToComponentsMap[data.Model.Registrant][data.Model.Model][comp.Component.Kind] = true + } + } else { + entriesToBeAddedInCompSheet = addEntriesInCompUpdateList(data.Model, data.Components, entriesToBeAddedInCompSheet) + } + } else { + + RegistrantToModelsToComponentsMap[data.Model.Registrant] = make(map[string]map[string]bool) + entriesToBeAddedInCompSheet = addEntriesInCompUpdateList(data.Model, data.Components, entriesToBeAddedInCompSheet) + } + } + + if modelBatchSize <= 0 { + // update model spreadsheet + err := updateModelsSheet(srv, cred, sheetId, entriesToBeAddedInModelSheet) + // Reset the list + entriesToBeAddedInModelSheet = []*ModelCSV{} + if err != nil { + Log.Error(err) + } + } + + if compBatchSize <= 0 { + // update comp spreadsheet + err := updateComponentsSheet(srv, cred, sheetId, entriesToBeAddedInCompSheet) + // Reset the list + entriesToBeAddedInCompSheet = []*ComponentCSV{} + entriesToBeAddedInModelSheet = []*ModelCSV{} + if err != nil { + Log.Error(err) + } + } + } + + if len(entriesToBeAddedInModelSheet) > 0 { + err := updateModelsSheet(srv, cred, sheetId, entriesToBeAddedInModelSheet) + if err != nil { + Log.Error(err) + } + } + + if len(entriesToBeAddedInCompSheet) > 0 { + err := updateComponentsSheet(srv, cred, sheetId, entriesToBeAddedInCompSheet) + if err != nil { + Log.Error(err) + } + return + } +} + +func updateModelsSheet(srv *sheets.Service, cred, sheetId string, values []*ModelCSV) error { + marshalledValues, err := marshalStructToCSValues[ModelCSV](values) + if err != nil { + return err + } + Log.Info("Appending", len(marshalledValues), "in the models sheet") + err = appendSheet(srv, cred, sheetId, ModelsSheetAppendRange, marshalledValues) + + return err +} + +func updateComponentsSheet(srv *sheets.Service, cred, sheetId string, values []*ComponentCSV) error { + marshalledValues, err := marshalStructToCSValues[ComponentCSV](values) + Log.Info("Appending", len(marshalledValues), "in the components sheet") + if err != nil { + return err + } + err = appendSheet(srv, cred, sheetId, ComponentsSheetAppendRange, marshalledValues) + + return err +} + +func appendSheet(srv *sheets.Service, cred, sheetId, appendRange string, values [][]interface{}) error { + + if len(values) == 0 { + return nil + } + _, err := srv.Spreadsheets.Values.Append(sheetId, appendRange, &sheets.ValueRange{ + MajorDimension: "ROWS", + Range: appendRange, + Values: values, + }).InsertDataOption("INSERT_ROWS").ValueInputOption("USER_ENTERED").Context(context.Background()).Do() + + if err != nil { + return ErrAppendToSheet(err, sheetId) + } + return nil +} + +func marshalStructToCSValues[K any](data []*K) ([][]interface{}, error) { + csvString, err := gocsv.MarshalString(data) + if err != nil { + return nil, ErrMarshalStructToCSV(err) + } + csvReader := bytes.NewBufferString(csvString) + decodedCSV, err := cuecsv.Decode(csvReader) + + if err != nil { + return nil, ErrMarshalStructToCSV(err) + } + + results := make([][]interface{}, 0) + + // The ouput is [ [col-names...] [row1][row2]] + // delete the first entry i.e. [col-names..], as it contains the column names and is not required as we are concerened only with rows + if len(decodedCSV) > 0 { + for idx, val := range decodedCSV { + if idx == 0 { + continue + } + result := make([]interface{}, 0, cap(val)) + for _, r := range val { + result = append(result, r) + } + results = append(results, result) + } + return results, nil + } + + return results, nil +} diff --git a/utils/registry/types.go b/utils/registry/types.go new file mode 100644 index 00000000..03b700bc --- /dev/null +++ b/utils/registry/types.go @@ -0,0 +1,61 @@ +package registry + +import ( + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" +) + +type SpreadsheetData struct { + Model *ModelCSV + Components []v1beta1.ComponentDefinition +} + +type ModelCSV struct { + Registrant string `json:"registrant" csv:"registrant"` + ModelDisplayName string `json:"modelDisplayName" csv:"modelDisplayName"` + Model string `json:"model" csv:"model"` + Category string `json:"category" csv:"category"` + SubCategory string `json:"subCategory" csv:"subCategory"` + Description string `json:"description" csv:"description"` + SourceURL string `json:"sourceURL" csv:"sourceURL"` + Website string `json:"website" csv:"website"` + Docs string `json:"docs" csv:"docs"` + Shape string `json:"shape" csv:"shape"` + PrimaryColor string `json:"primaryColor" csv:"primaryColor"` + SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` + StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` + Styles string `json:"styles" csv:"styles"` + ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` + DefaultData string `json:"defaultData" csv:"defaultData"` + Capabilities string `json:"capabilities" csv:"capabilities"` + LogoURL string `json:"logoURL" csv:"logoURL"` + SVGColor string `json:"svgColor" csv:"svgColor"` + SVGWhite string `json:"svgWhite" csv:"svgWhite"` + SVGComplete string `json:"svgComplete" csv:"svgComplete"` + IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` + PublishToRegistry string `json:"publishToRegistry" csv:"publishToRegistry"` + AboutProject string `json:"aboutProject" csv:"-"` + PageSubtTitle string `json:"pageSubtitle" csv:"-"` + DocsURL string `json:"docsURL" csv:"-"` + StandardBlurb string `json:"standardBlurb" csv:"-"` + Feature1 string `json:"feature1" csv:"-"` + Feature2 string `json:"feature2" csv:"-"` + Feature3 string `json:"feature3" csv:"-"` + HowItWorks string `json:"howItWorks" csv:"-"` + HowItWorksDetails string `json:"howItWorksDetails" csv:"-"` + Screenshots string `json:"screenshots" csv:"-"` + FullPage string `json:"fullPage" csv:"-"` + PublishToSites string `json:"publishToSites" csv:"-"` +} + +type ModelCSVHelper struct { + SpreadsheetID int64 + SpreadsheetURL string + Title string + CSVPath string + Models []ModelCSV +} + +type CompGenerateTracker struct { + TotalComps int + Version string +} \ No newline at end of file From 1ef6ff99f4260b670e0527bdf9c16a761f6eb560 Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Tue, 23 Jul 2024 19:26:35 +0900 Subject: [PATCH 3/6] chore: remove unused Signed-off-by: dusdjhyeon --- utils/registry/helper.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/registry/helper.go b/utils/registry/helper.go index 15770076..2c97a623 100644 --- a/utils/registry/helper.go +++ b/utils/registry/helper.go @@ -7,7 +7,6 @@ import ( "strings" "path/filepath" "google.golang.org/api/sheets/v4" - "github.com/layer5io/meshkit/utils/store" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/csv" "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -18,7 +17,6 @@ var ( rowIndex = 1 shouldRegisterColIndex = -1 shouldRegisterMod = "publishToSites" - modelToCompGenerateTracker = store.NewGenericThreadSafeStore[CompGenerateTracker]() ) var modelMetadataValues = []string{ From d20778ee3ad179220c38185c1553aea68fa89c98 Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Wed, 24 Jul 2024 14:44:43 +0900 Subject: [PATCH 4/6] feat: add meshery utils model to meshkit Signed-off-by: dusdjhyeon --- utils/registry/component.go | 12 +- utils/registry/error.go | 2 +- utils/registry/helper.go | 179 -------------- utils/registry/model.go | 354 +++++++++++++++++++++++++++ utils/registry/spreadsheet_update.go | 9 + utils/registry/types.go | 48 +--- 6 files changed, 372 insertions(+), 232 deletions(-) delete mode 100644 utils/registry/helper.go create mode 100644 utils/registry/model.go diff --git a/utils/registry/component.go b/utils/registry/component.go index 9869c1ae..ffa37b87 100644 --- a/utils/registry/component.go +++ b/utils/registry/component.go @@ -7,21 +7,23 @@ import ( "strconv" "strings" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/logger" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/csv" "github.com/layer5io/meshkit/utils/manifests" ) const ( - SVG_WIDTH = 20 - SVG_HEIGHT = 20 + SVG_WIDTH = 20 + SVG_HEIGHT = 20 + rowIndex = 1 + shouldRegisterColIndex = -1 ) var ( - Log logger.Handler - LogError logger.Handler + Log logger.Handler + LogError logger.Handler ) type ComponentCSV struct { diff --git a/utils/registry/error.go b/utils/registry/error.go index 890b5987..a50a7766 100644 --- a/utils/registry/error.go +++ b/utils/registry/error.go @@ -48,4 +48,4 @@ func ErrMarshalStructToCSV(err error) error { []string{err.Error()}, []string{"The column names in your spreadsheet do not match the names in the struct.", " For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}, []string{"The column names in the spreadsheet do not match the names in the struct. Please make sure they are spelled exactly the same and use the same case (uppercase/lowercase).", "The value you are trying to convert is not of the expected type for the column. Please ensure it is a [number, string, date, etc.].", "The column names in your spreadsheet do not match the names in the struct. For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}) -} \ No newline at end of file +} diff --git a/utils/registry/helper.go b/utils/registry/helper.go deleted file mode 100644 index 2c97a623..00000000 --- a/utils/registry/helper.go +++ /dev/null @@ -1,179 +0,0 @@ -package registry - -import ( - "fmt" - "os" - "strconv" - "strings" - "path/filepath" - "google.golang.org/api/sheets/v4" - "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/csv" - "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" -) - -var ( - rowIndex = 1 - shouldRegisterColIndex = -1 - shouldRegisterMod = "publishToSites" -) - -var modelMetadataValues = []string{ - "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "isAnnotation", "shape", -} - -func NewModelCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ModelCSVHelper, error) { - sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) - Log.Info("Downloading CSV from: ", sheetURL) - dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") - err := os.MkdirAll(dirPath, 0755) - if err != nil { - return nil, ErrCreateDir(err, dirPath) - } - csvPath := filepath.Join(dirPath, "models.csv") - err = utils.DownloadFile(csvPath, sheetURL) - if err != nil { - return nil, utils.ErrReadingRemoteFile(err) - } - - return &ModelCSVHelper{ - SpreadsheetID: spreadsheetID, - SpreadsheetURL: sheetURL, - Models: []ModelCSV{}, - CSVPath: csvPath, - Title: spreadsheetName, - }, nil -} - -func GetSheetIDFromTitle(s *sheets.Spreadsheet, title string) int64 { - for _, sheet := range s.Sheets { - if sheet.Properties.Title == title { - return sheet.Properties.SheetId - } - } - return -1 -} - -func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { - ch := make(chan ModelCSV, 1) - errorChan := make(chan error, 1) - csvReader, err := csv.NewCSVParser[ModelCSV](mch.CSVPath, rowIndex, nil, func(columns []string, currentRow []string) bool { - index := 0 - - if parseForDocs { - index = GetIndexForRegisterCol(columns, shouldRegisterMod) - } else { - // Generation of models should not consider publishedToRegistry column value. - // Generation should happen for all models, while during registration "published" attribute should be respected. - return true - } - if index != -1 && index < len(currentRow) { - shouldRegister := currentRow[index] - return strings.ToLower(shouldRegister) == "true" - } - return false - }) - - if err != nil { - return ErrFileRead(err) - } - - go func() { - Log.Info("Parsing Models...") - err := csvReader.Parse(ch, errorChan) - if err != nil { - errorChan <- err - } - }() - for { - select { - - case data := <-ch: - mch.Models = append(mch.Models, data) - Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s]", data.Registrant, data.Model)) - case err := <-errorChan: - return ErrFileRead(err) - - case <-csvReader.Context.Done(): - return nil - } - } -} - -func (mcv *ModelCSV) CreateModelDefinition(version, defVersion string) v1beta1.Model { - status := entity.Ignored - if strings.ToLower(mcv.PublishToRegistry) == "true" { - status = entity.Enabled - } - - model := v1beta1.Model{ - VersionMeta: v1beta1.VersionMeta{ - Version: defVersion, - SchemaVersion: v1beta1.ModelSchemaVersion, - }, - Name: mcv.Model, - DisplayName: mcv.ModelDisplayName, - Status: status, - Registrant: v1beta1.Host{ - Hostname: utils.ReplaceSpacesAndConvertToLowercase(mcv.Registrant), - }, - Category: v1beta1.Category{ - Name: mcv.Category, - }, - SubCategory: mcv.SubCategory, - Model: v1beta1.ModelEntity{ - Version: version, - }, - } - err := mcv.UpdateModelDefinition(&model) - if err != nil { - Log.Error(err) - } - return model -} - -func (m *ModelCSV) UpdateModelDefinition(modelDef *v1beta1.Model) error { - - metadata := map[string]interface{}{} - modelMetadata, err := utils.MarshalAndUnmarshal[ModelCSV, map[string]interface{}](*m) - if err != nil { - return err - } - metadata = utils.MergeMaps(metadata, modelDef.Metadata) - - for _, key := range modelMetadataValues { - if key == "svgColor" || key == "svgWhite" { - svg, err := utils.Cast[string](modelMetadata[key]) - if err == nil { - metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) - if err != nil { - // If svg cannot be updated, assign the svg value as it is - metadata[key] = modelMetadata[key] - } - } - } - metadata[key] = modelMetadata[key] - } - - isAnnotation := false - if strings.ToLower(m.IsAnnotation) == "true" { - isAnnotation = true - } - metadata["isAnnotation"] = isAnnotation - modelDef.Metadata = metadata - return nil -} - -func GetIndexForRegisterCol(cols []string, shouldRegister string) int { - if shouldRegisterColIndex != -1 { - return shouldRegisterColIndex - } - - for index, col := range cols { - if col == shouldRegister { - return index - } - } - return shouldRegisterColIndex -} \ No newline at end of file diff --git a/utils/registry/model.go b/utils/registry/model.go new file mode 100644 index 00000000..5e8a76a4 --- /dev/null +++ b/utils/registry/model.go @@ -0,0 +1,354 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/csv" +) + +var ( + shouldRegisterMod = "publishToSites" +) + +type ModelCSV struct { + Registrant string `json:"registrant" csv:"registrant"` + ModelDisplayName string `json:"modelDisplayName" csv:"modelDisplayName"` + Model string `json:"model" csv:"model"` + Category string `json:"category" csv:"category"` + SubCategory string `json:"subCategory" csv:"subCategory"` + Description string `json:"description" csv:"description"` + SourceURL string `json:"sourceURL" csv:"sourceURL"` + Website string `json:"website" csv:"website"` + Docs string `json:"docs" csv:"docs"` + Shape string `json:"shape" csv:"shape"` + PrimaryColor string `json:"primaryColor" csv:"primaryColor"` + SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` + StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` + Styles string `json:"styles" csv:"styles"` + ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` + DefaultData string `json:"defaultData" csv:"defaultData"` + Capabilities string `json:"capabilities" csv:"capabilities"` + LogoURL string `json:"logoURL" csv:"logoURL"` + SVGColor string `json:"svgColor" csv:"svgColor"` + SVGWhite string `json:"svgWhite" csv:"svgWhite"` + SVGComplete string `json:"svgComplete" csv:"svgComplete"` + IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` + PublishToRegistry string `json:"publishToRegistry" csv:"publishToRegistry"` + AboutProject string `json:"aboutProject" csv:"-"` + PageSubtTitle string `json:"pageSubtitle" csv:"-"` + DocsURL string `json:"docsURL" csv:"-"` + StandardBlurb string `json:"standardBlurb" csv:"-"` + Feature1 string `json:"feature1" csv:"-"` + Feature2 string `json:"feature2" csv:"-"` + Feature3 string `json:"feature3" csv:"-"` + HowItWorks string `json:"howItWorks" csv:"-"` + HowItWorksDetails string `json:"howItWorksDetails" csv:"-"` + Screenshots string `json:"screenshots" csv:"-"` + FullPage string `json:"fullPage" csv:"-"` + PublishToSites string `json:"publishToSites" csv:"-"` +} + +var modelMetadataValues = []string{ + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "isAnnotation", "shape", +} + +func (m *ModelCSV) UpdateModelDefinition(modelDef *v1beta1.Model) error { + + metadata := map[string]interface{}{} + modelMetadata, err := utils.MarshalAndUnmarshal[ModelCSV, map[string]interface{}](*m) + if err != nil { + return err + } + metadata = utils.MergeMaps(metadata, modelDef.Metadata) + + for _, key := range modelMetadataValues { + if key == "svgColor" || key == "svgWhite" { + svg, err := utils.Cast[string](modelMetadata[key]) + if err == nil { + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) + if err != nil { + // If svg cannot be updated, assign the svg value as it is + metadata[key] = modelMetadata[key] + } + } + } + metadata[key] = modelMetadata[key] + } + + isAnnotation := false + if strings.ToLower(m.IsAnnotation) == "true" { + isAnnotation = true + } + metadata["isAnnotation"] = isAnnotation + modelDef.Metadata = metadata + return nil +} + +func (mcv *ModelCSV) CreateModelDefinition(version, defVersion string) v1beta1.Model { + status := entity.Ignored + if strings.ToLower(mcv.PublishToRegistry) == "true" { + status = entity.Enabled + } + + model := v1beta1.Model{ + VersionMeta: v1beta1.VersionMeta{ + Version: defVersion, + SchemaVersion: v1beta1.ModelSchemaVersion, + }, + Name: mcv.Model, + DisplayName: mcv.ModelDisplayName, + Status: status, + Registrant: v1beta1.Host{ + Hostname: utils.ReplaceSpacesAndConvertToLowercase(mcv.Registrant), + }, + Category: v1beta1.Category{ + Name: mcv.Category, + }, + SubCategory: mcv.SubCategory, + Model: v1beta1.ModelEntity{ + Version: version, + }, + } + err := mcv.UpdateModelDefinition(&model) + if err != nil { + Log.Error(err) + } + return model +} + +type ModelCSVHelper struct { + SpreadsheetID int64 + SpreadsheetURL string + Title string + CSVPath string + Models []ModelCSV +} + +func NewModelCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ModelCSVHelper, error) { + sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) + Log.Info("Downloading CSV from: ", sheetURL) + dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return nil, utils.ErrCreateDir(err, dirPath) + } + csvPath := filepath.Join(dirPath, "models.csv") + err = utils.DownloadFile(csvPath, sheetURL) + if err != nil { + return nil, utils.ErrReadingRemoteFile(err) + } + + return &ModelCSVHelper{ + SpreadsheetID: spreadsheetID, + SpreadsheetURL: sheetURL, + Models: []ModelCSV{}, + CSVPath: csvPath, + Title: spreadsheetName, + }, nil +} + +func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { + ch := make(chan ModelCSV, 1) + errorChan := make(chan error, 1) + csvReader, err := csv.NewCSVParser[ModelCSV](mch.CSVPath, rowIndex, nil, func(columns []string, currentRow []string) bool { + index := 0 + + if parseForDocs { + index = GetIndexForRegisterCol(columns, shouldRegisterMod) + } else { + // Generation of models should not consider publishedToRegistry column value. + // Generation should happen for all models, while during registration "published" attribute should be respected. + return true + } + if index != -1 && index < len(currentRow) { + shouldRegister := currentRow[index] + return strings.ToLower(shouldRegister) == "true" + } + return false + }) + + if err != nil { + return ErrFileRead(err) + } + + go func() { + Log.Info("Parsing Models...") + err := csvReader.Parse(ch, errorChan) + if err != nil { + errorChan <- err + } + }() + for { + select { + + case data := <-ch: + mch.Models = append(mch.Models, data) + Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s]", data.Registrant, data.Model)) + case err := <-errorChan: + return ErrFileRead(err) + + case <-csvReader.Context.Done(): + return nil + } + } +} + +func GetIndexForRegisterCol(cols []string, shouldRegister string) int { + if shouldRegisterColIndex != -1 { + return shouldRegisterColIndex + } + + for index, col := range cols { + if col == shouldRegister { + return index + } + } + return shouldRegisterColIndex +} + +func (m ModelCSV) CreateMarkDownForMDXStyle(componentsMetadata string) string { + formattedName := utils.FormatName(m.Model) + var template string = `--- +title: %s +subtitle: %s +integrationIcon: icons/color/%s-color.svg +darkModeIntegrationIcon: icons/white/%s-white.svg +docURL: %s +description: %s +category: %s +subcategory: %s +registrant: %s +components: %v +featureList: [ + "%s", + "%s", + "%s" +] +workingSlides: [ + %s, + %s +] +howItWorks: "%s" +howItWorksDetails: "%s" +published: %s +--- +

+%s +

+%s +` + markdown := fmt.Sprintf(template, + m.ModelDisplayName, + m.PageSubtTitle, + formattedName, + formattedName, + m.DocsURL, + m.Description, + m.Category, + m.SubCategory, + m.Registrant, + componentsMetadata, + m.Feature1, + m.Feature2, + m.Feature3, + `../_images/meshmap-visualizer.png`, + `../_images/meshmap-designer.png`, + m.HowItWorks, + m.HowItWorksDetails, + m.PublishToSites, + m.AboutProject, + m.StandardBlurb, + ) + markdown = strings.ReplaceAll(markdown, "\r", "\n") + return markdown +} + +// Creates JSON formatted meshmodel attribute item for JSON Style docs +func (m ModelCSV) CreateJSONItem(iconDir string) string { + formattedModelName := utils.FormatName(m.Model) + json := "{" + json += fmt.Sprintf("\"name\":\"%s\"", m.Model) + // If SVGs exist, then add the paths to json + if m.SVGColor != "" { + json += fmt.Sprintf(",\"color\":\"%s/icons/color/%s-color.svg\"", iconDir, formattedModelName) + } + + if m.SVGWhite != "" { + json += fmt.Sprintf(",\"white\":\"%s/icons/white/%s-white.svg\"", iconDir, formattedModelName) + } + + json += fmt.Sprintf(",\"permalink\":\"%s\"", m.DocsURL) + + json += "}" + return json +} + +func (m ModelCSV) CreateMarkDownForMDStyle(componentsMetadata string) string { + formattedName := utils.FormatName(m.Model) + + var template string = `--- +layout: integration +title: %s +subtitle: %s +image: /assets/img/integrations/%s/icons/color/%s-color.svg +permalink: extensibility/integrations/%s +docURL: %s +description: %s +integrations-category: %s +integrations-subcategory: %s +registrant: %s +components: %v +featureList: [ + "%s", + "%s", + "%s" +] +howItWorks: "%s" +howItWorksDetails: "%s" +language: en +list: include +type: extensibility +category: integrations +--- +` + markdown := fmt.Sprintf(template, + m.ModelDisplayName, + m.PageSubtTitle, + formattedName, + formattedName, + formattedName, + m.DocsURL, + m.Description, + m.Category, + m.SubCategory, + m.Registrant, + componentsMetadata, + m.Feature1, + m.Feature2, + m.Feature3, + m.HowItWorks, + m.HowItWorksDetails, + ) + + markdown = strings.ReplaceAll(markdown, "\r", "\n") + + return markdown +} + +func (m ModelCSVHelper) Cleanup() error { + // remove csv file + Log.Info("Removing CSV file: ", m.CSVPath) + err := os.Remove(m.CSVPath) + if err != nil { + return err + } + + return nil +} diff --git a/utils/registry/spreadsheet_update.go b/utils/registry/spreadsheet_update.go index b1f5716a..52cae39c 100644 --- a/utils/registry/spreadsheet_update.go +++ b/utils/registry/spreadsheet_update.go @@ -222,3 +222,12 @@ func marshalStructToCSValues[K any](data []*K) ([][]interface{}, error) { return results, nil } + +func GetSheetIDFromTitle(s *sheets.Spreadsheet, title string) int64 { + for _, sheet := range s.Sheets { + if sheet.Properties.Title == title { + return sheet.Properties.SheetId + } + } + return -1 +} diff --git a/utils/registry/types.go b/utils/registry/types.go index 03b700bc..29d00a18 100644 --- a/utils/registry/types.go +++ b/utils/registry/types.go @@ -9,53 +9,7 @@ type SpreadsheetData struct { Components []v1beta1.ComponentDefinition } -type ModelCSV struct { - Registrant string `json:"registrant" csv:"registrant"` - ModelDisplayName string `json:"modelDisplayName" csv:"modelDisplayName"` - Model string `json:"model" csv:"model"` - Category string `json:"category" csv:"category"` - SubCategory string `json:"subCategory" csv:"subCategory"` - Description string `json:"description" csv:"description"` - SourceURL string `json:"sourceURL" csv:"sourceURL"` - Website string `json:"website" csv:"website"` - Docs string `json:"docs" csv:"docs"` - Shape string `json:"shape" csv:"shape"` - PrimaryColor string `json:"primaryColor" csv:"primaryColor"` - SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` - StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` - Styles string `json:"styles" csv:"styles"` - ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` - DefaultData string `json:"defaultData" csv:"defaultData"` - Capabilities string `json:"capabilities" csv:"capabilities"` - LogoURL string `json:"logoURL" csv:"logoURL"` - SVGColor string `json:"svgColor" csv:"svgColor"` - SVGWhite string `json:"svgWhite" csv:"svgWhite"` - SVGComplete string `json:"svgComplete" csv:"svgComplete"` - IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` - PublishToRegistry string `json:"publishToRegistry" csv:"publishToRegistry"` - AboutProject string `json:"aboutProject" csv:"-"` - PageSubtTitle string `json:"pageSubtitle" csv:"-"` - DocsURL string `json:"docsURL" csv:"-"` - StandardBlurb string `json:"standardBlurb" csv:"-"` - Feature1 string `json:"feature1" csv:"-"` - Feature2 string `json:"feature2" csv:"-"` - Feature3 string `json:"feature3" csv:"-"` - HowItWorks string `json:"howItWorks" csv:"-"` - HowItWorksDetails string `json:"howItWorksDetails" csv:"-"` - Screenshots string `json:"screenshots" csv:"-"` - FullPage string `json:"fullPage" csv:"-"` - PublishToSites string `json:"publishToSites" csv:"-"` -} - -type ModelCSVHelper struct { - SpreadsheetID int64 - SpreadsheetURL string - Title string - CSVPath string - Models []ModelCSV -} - type CompGenerateTracker struct { TotalComps int Version string -} \ No newline at end of file +} From cfae90040fdd194adc9ee958cccdca5241e8fd08 Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Wed, 31 Jul 2024 09:40:26 +0900 Subject: [PATCH 5/6] fix: refactoring err code Signed-off-by: dusdjhyeon --- generators/artifacthub/error.go | 8 ++------ generators/artifacthub/package.go | 2 +- generators/registry.go | 2 +- models/meshmodel/registry/error.go | 14 +++++++------- models/patterns/error.go | 2 +- utils/error.go | 15 ++++++++++----- utils/registry/component.go | 2 +- utils/registry/error.go | 25 +++---------------------- utils/registry/model.go | 4 ++-- 9 files changed, 28 insertions(+), 46 deletions(-) diff --git a/generators/artifacthub/error.go b/generators/artifacthub/error.go index fb4c1a91..533d6f01 100644 --- a/generators/artifacthub/error.go +++ b/generators/artifacthub/error.go @@ -9,10 +9,9 @@ import ( var ( ErrGetChartUrlCode = "meshkit-11134" ErrGetAhPackageCode = "meshkit-11135" - ErrComponentGenerateCode = "meshkit-11136" ErrGetAllHelmPackagesCode = "meshkit-11137" - ErrChartUrlEmptyCode = "replace_me" - ErrNoPackageFoundCode = "replace_me" + ErrChartUrlEmptyCode = "meshkit-11245" + ErrNoPackageFoundCode = "meshkit-11246" ) func ErrGetAllHelmPackages(err error) error { @@ -27,9 +26,6 @@ func ErrGetAhPackage(err error) error { return errors.New(ErrGetAhPackageCode, errors.Alert, []string{"Could not get the ArtifactHub package with the given name"}, []string{err.Error()}, []string{""}, []string{"make sure that the package exists"}) } -func ErrComponentGenerate(err error) error { - return errors.New(ErrComponentGenerateCode, errors.Alert, []string{"failed to generate components for the package"}, []string{err.Error()}, []string{}, []string{"Make sure that the package is compatible"}) -} func ErrChartUrlEmpty(modelName string, registrantName string) error { return errors.New( ErrChartUrlEmptyCode, diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 00bb416d..a99a2c16 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -45,7 +45,7 @@ func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) } crds, err := manifests.GetCrdsFromHelm(pkg.ChartUrl) if err != nil { - return components, ErrComponentGenerate(err) + return components, utils.ErrComponentGenerate(err) } for _, crd := range crds { comp, err := component.Generate(crd) diff --git a/generators/registry.go b/generators/registry.go index 470989f3..88c4bbf6 100644 --- a/generators/registry.go +++ b/generators/registry.go @@ -309,7 +309,7 @@ func GenerateDefsForCoreRegistrant(model registry.ModelCSV) error { err = componentDef.WriteComponentDefinition(compDirPath) if err != nil { - err = registry.ErrGenerateComponent(err, model.Model, componentDef.DisplayName) + err = utils.ErrComponentGenerate(err) LogError.Error(err) } return nil diff --git a/models/meshmodel/registry/error.go b/models/meshmodel/registry/error.go index 6046a223..8d2682ac 100644 --- a/models/meshmodel/registry/error.go +++ b/models/meshmodel/registry/error.go @@ -7,13 +7,13 @@ import ( ) var ( - ErrUnknownHostCode = "replace_me" - ErrEmptySchemaCode = "replace_me" - ErrMarshalingRegisteryAttemptsCode = "replace_me" - ErrWritingRegisteryAttemptsCode = "replace_me" - ErrRegisteringEntityCode = "replace_me" - ErrUnknownHostInMapCode = "replace_me" - ErrCreatingUserDataDirectoryCode = "replace_me" + ErrUnknownHostCode = "meshkit-11247" + ErrEmptySchemaCode = "meshkit-11248" + ErrMarshalingRegisteryAttemptsCode = "meshkit-11249" + ErrWritingRegisteryAttemptsCode = "meshkit-11250" + ErrRegisteringEntityCode = "meshkit-11251" + ErrUnknownHostInMapCode = "meshkit-11252" + ErrCreatingUserDataDirectoryCode = "meshkit-11253" ) func ErrUnknownHost(err error) error { diff --git a/models/patterns/error.go b/models/patterns/error.go index ad37aad6..03a95ab7 100644 --- a/models/patterns/error.go +++ b/models/patterns/error.go @@ -3,7 +3,7 @@ package patterns import "github.com/layer5io/meshkit/errors" const ( - ErrInvalidVersionCode = "" + ErrInvalidVersionCode = "meshkit-11254" ) func ErrInvalidVersion(err error) error { diff --git a/utils/error.go b/utils/error.go index 8e9a42cf..1fae0700 100644 --- a/utils/error.go +++ b/utils/error.go @@ -40,12 +40,13 @@ var ( ErrExtractTarXZCode = "meshkit-11184" ErrExtractZipCode = "meshkit-11185" ErrReadDirCode = "meshkit-11186" - ErrInvalidSchemaVersionCode = "replace_me" - ErrFileWalkDirCode = "replace_me" - ErrRelPathCode = "replace_me" - ErrCopyFileCode = "replace_me" - ErrCloseFileCode = "replace_me" + ErrInvalidSchemaVersionCode = "meshkit-11255" + ErrFileWalkDirCode = "meshkit-11256" + ErrRelPathCode = "meshkit-11257" + ErrCopyFileCode = "meshkit-11258" + ErrCloseFileCode = "meshkit-11259" ErrCompressToTarGZCode = "meshkit-11248" + ErrComponentGenerateCode = "meshkit-11260" ) var ( ErrExtractType = errors.New( @@ -66,6 +67,10 @@ var ( ) ) +func ErrComponentGenerate(err error) error { + return errors.New(ErrComponentGenerateCode, errors.Alert, []string{"failed to generate components for the package"}, []string{err.Error()}, []string{}, []string{"Make sure that the package is compatible"}) +} + func ErrCueLookup(err error) error { return errors.New(ErrCueLookupCode, errors.Alert, []string{"Could not lookup the given path in the CUE value"}, []string{err.Error()}, []string{""}, []string{"make sure that the path is a valid cue expression and is correct", "make sure that there exists a field with the given path", "make sure that the given root value is correct"}) } diff --git a/utils/registry/component.go b/utils/registry/component.go index ffa37b87..005fb0a2 100644 --- a/utils/registry/component.go +++ b/utils/registry/component.go @@ -154,7 +154,7 @@ func (mch *ComponentCSVHelper) ParseComponentsSheet() error { }) if err != nil { - return ErrFileRead(err) + return utils.ErrReadFile(err, mch.CSVPath) } go func() { diff --git a/utils/registry/error.go b/utils/registry/error.go index a50a7766..ab5c9309 100644 --- a/utils/registry/error.go +++ b/utils/registry/error.go @@ -6,34 +6,15 @@ import ( ) var ( - ErrAppendToSheetCode = "mesheryctl-1116" - ErrCreateDirCode = "mesheryctl-1066" - ErrGenerateComponentCode = "mesheryctl-1056" - ErrGenerateModelCode = "mesheryctl-1055" - ErrFileReadCode = "mesheryctl-1095" - ErrMarshalStructToCSVCode = "mesheryctl-1115" + ErrAppendToSheetCode = "meshkit-11261" + ErrGenerateModelCode = "meshkit-11262" + ErrMarshalStructToCSVCode = "meshkit-11263" ) func ErrGenerateModel(err error, modelName string) error { return errors.New(ErrGenerateModelCode, errors.Alert, []string{fmt.Sprintf("error generating model: %s", modelName)}, []string{fmt.Sprintf("Error generating model: %s\n %s", modelName, err.Error())}, []string{"Registrant used for the model is not supported", "Verify the model's source URL.", "Failed to create a local directory in the filesystem for this model."}, []string{"Ensure that each kind of registrant used is a supported kind.", "Ensure correct model source URL is provided and properly formatted.", "Ensure sufficient permissions to allow creation of model directory."}) } -func ErrCreateDir(err error, obj string) error { - return errors.New(ErrCreateDirCode, errors.Alert, []string{"Error creating directory ", obj}, []string{err.Error()}, []string{}, []string{}) -} - -func ErrGenerateComponent(err error, modelName, compName string) error { - return errors.New(ErrGenerateComponentCode, errors.Alert, []string{"error generating comp %s of model %s", compName, modelName}, []string{err.Error()}, []string{}, []string{}) -} - -func ErrFileRead(err error) error { - return errors.New(ErrFileReadCode, errors.Alert, - []string{"File read error"}, - []string{err.Error()}, - []string{"The provided file is not present or has an invalid path."}, - []string{"To proceed, provide a valid file path with a valid file."}) -} - func ErrAppendToSheet(err error, id string) error { return errors.New(ErrAppendToSheetCode, errors.Alert, []string{fmt.Sprintf("Failed to append data into sheet %s", id)}, diff --git a/utils/registry/model.go b/utils/registry/model.go index 5e8a76a4..c26e40be 100644 --- a/utils/registry/model.go +++ b/utils/registry/model.go @@ -175,7 +175,7 @@ func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { }) if err != nil { - return ErrFileRead(err) + return utils.ErrReadFile(err, mch.CSVPath) } go func() { @@ -192,7 +192,7 @@ func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { mch.Models = append(mch.Models, data) Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s]", data.Registrant, data.Model)) case err := <-errorChan: - return ErrFileRead(err) + return utils.ErrReadFile(err, mch.CSVPath) case <-csvReader.Context.Done(): return nil From af50f7598db623be7d03e38ccdcee609f885ef2c Mon Sep 17 00:00:00 2001 From: dusdjhyeon Date: Sat, 3 Aug 2024 10:26:15 +0900 Subject: [PATCH 6/6] fix: applying the change Signed-off-by: dusdjhyeon --- helpers/component_info.json | 2 +- utils/registry/component.go | 2 +- utils/registry/model.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/component_info.json b/helpers/component_info.json index 92af709d..3fb70b72 100644 --- a/helpers/component_info.json +++ b/helpers/component_info.json @@ -1,5 +1,5 @@ { "name": "meshkit", "type": "library", - "next_error_code": 11245 + "next_error_code": 11264 } \ No newline at end of file diff --git a/utils/registry/component.go b/utils/registry/component.go index 005fb0a2..bb7c22a9 100644 --- a/utils/registry/component.go +++ b/utils/registry/component.go @@ -88,7 +88,7 @@ func (c *ComponentCSV) UpdateCompDefinition(compDef *v1beta1.ComponentDefinition if key == "svgColor" || key == "svgWhite" { svg, err := utils.Cast[string](compMetadata[key]) if err == nil { - metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT, false) if err != nil { // If svg cannot be updated, assign the svg value as it is metadata[key] = compMetadata[key] diff --git a/utils/registry/model.go b/utils/registry/model.go index c26e40be..681e72ba 100644 --- a/utils/registry/model.go +++ b/utils/registry/model.go @@ -72,7 +72,7 @@ func (m *ModelCSV) UpdateModelDefinition(modelDef *v1beta1.Model) error { if key == "svgColor" || key == "svgWhite" { svg, err := utils.Cast[string](modelMetadata[key]) if err == nil { - metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT) + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT, false) if err != nil { // If svg cannot be updated, assign the svg value as it is metadata[key] = modelMetadata[key]