diff --git a/generators/artifacthub/error.go b/generators/artifacthub/error.go index fb4c1a91..533d6f01 100644 --- a/generators/artifacthub/error.go +++ b/generators/artifacthub/error.go @@ -9,10 +9,9 @@ import ( var ( ErrGetChartUrlCode = "meshkit-11134" ErrGetAhPackageCode = "meshkit-11135" - ErrComponentGenerateCode = "meshkit-11136" ErrGetAllHelmPackagesCode = "meshkit-11137" - ErrChartUrlEmptyCode = "replace_me" - ErrNoPackageFoundCode = "replace_me" + ErrChartUrlEmptyCode = "meshkit-11245" + ErrNoPackageFoundCode = "meshkit-11246" ) func ErrGetAllHelmPackages(err error) error { @@ -27,9 +26,6 @@ func ErrGetAhPackage(err error) error { return errors.New(ErrGetAhPackageCode, errors.Alert, []string{"Could not get the ArtifactHub package with the given name"}, []string{err.Error()}, []string{""}, []string{"make sure that the package exists"}) } -func ErrComponentGenerate(err error) error { - return errors.New(ErrComponentGenerateCode, errors.Alert, []string{"failed to generate components for the package"}, []string{err.Error()}, []string{}, []string{"Make sure that the package is compatible"}) -} func ErrChartUrlEmpty(modelName string, registrantName string) error { return errors.New( ErrChartUrlEmptyCode, diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 00bb416d..a99a2c16 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -45,7 +45,7 @@ func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) } crds, err := manifests.GetCrdsFromHelm(pkg.ChartUrl) if err != nil { - return components, ErrComponentGenerate(err) + return components, utils.ErrComponentGenerate(err) } for _, crd := range crds { comp, err := component.Generate(crd) diff --git a/generators/registry.go b/generators/registry.go new file mode 100644 index 00000000..88c4bbf6 --- /dev/null +++ b/generators/registry.go @@ -0,0 +1,331 @@ +package generators + +import ( + "context" + "fmt" + "os" + "strconv" + "path/filepath" + "encoding/json" + "net/url" + "sync" + "time" + + "google.golang.org/api/sheets/v4" + "github.com/layer5io/meshkit/utils/walker" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/generators/github" + "github.com/layer5io/meshkit/utils/store" + "github.com/layer5io/meshkit/utils/registry" + "golang.org/x/sync/semaphore" + "github.com/layer5io/meshkit/logger" +) + +var ( + srv *sheets.Service + totalAggregateModel int + totalAggregateComponents int + componentSpredsheetGID int64 + sheetGID int64 + registryLocation string + logFile *os.File + errorLogFile *os.File + Log logger.Handler + LogError logger.Handler +) + +var ( + artifactHubCount = 0 + artifactHubRateLimit = 100 + artifactHubRateLimitDur = 5 * time.Minute + artifactHubMutex sync.Mutex + defVersion = "v1.0.0" + GoogleSpreadSheetURL = "https://docs.google.com/spreadsheets/d/" + logDirPath = filepath.Join(utils.GetHome(), ".meshery", "logs", "registry") + modelToCompGenerateTracker = store.NewGenericThreadSafeStore[registry.CompGenerateTracker]() +) + +func InvokeGenerationFromSheet(wg *sync.WaitGroup, spreadsheetID string, spreadsheetCred string) error { + + weightedSem := semaphore.NewWeighted(20) + url := GoogleSpreadSheetURL + spreadsheetID + totalAvailableModels := 0 + spreadsheeetChan := make(chan registry.SpreadsheetData) + + defer func() { + logModelGenerationSummary(modelToCompGenerateTracker) + + Log.UpdateLogOutput(os.Stdout) + Log.UpdateLogOutput(os.Stdout) + Log.Info(fmt.Sprintf("Summary: %d models, %d components generated.", totalAggregateModel, totalAggregateComponents)) + + Log.Info("See ", logDirPath, " for detailed logs.") + + _ = logFile.Close() + _ = errorLogFile.Close() + totalAggregateModel = 0 + totalAggregateComponents = 0 + }() + + modelCSVHelper, err := parseModelSheet(url) + if err != nil { + return err + } + + componentCSVHelper, err := parseComponentSheet(url) + if err != nil { + return err + } + + Log.UpdateLogOutput(logFile) + Log.UpdateLogOutput(errorLogFile) + var wgForSpreadsheetUpdate sync.WaitGroup + wgForSpreadsheetUpdate.Add(1) + go func() { + registry.ProcessModelToComponentsMap(componentCSVHelper.Components) + registry.VerifyandUpdateSpreadsheet(spreadsheetCred, &wgForSpreadsheetUpdate, srv, spreadsheeetChan, spreadsheetID) + }() + + // Iterate models from the spreadsheet + for _, model := range modelCSVHelper.Models { + totalAvailableModels++ + + ctx := context.Background() + + err := weightedSem.Acquire(ctx, 1) + if err != nil { + break + } + + wg.Add(1) + go func(model registry.ModelCSV) { + defer func() { + wg.Done() + weightedSem.Release(1) + }() + if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "meshery" { + err = GenerateDefsForCoreRegistrant(model) + if err != nil { + LogError.Error(err) + } + return + } + + generator, err := NewGenerator(model.Registrant, model.SourceURL, model.Model) + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + + if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "artifacthub" { + rateLimitArtifactHub() + + } + pkg, err := generator.GetPackage() + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + + version := pkg.GetVersion() + modelDirPath, compDirPath, err := CreateVersionedDirectoryForModelAndComp(version, model.Model) + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + modelDef, err := writeModelDefToFileSystem(&model, version, modelDirPath) + if err != nil { + LogError.Error(err) + return + } + + comps, err := pkg.GenerateComponents() + if err != nil { + LogError.Error(registry.ErrGenerateModel(err, model.Model)) + return + } + Log.Info("Current model: ", model.Model) + Log.Info(" extracted ", len(comps), " components for ", model.ModelDisplayName, " (", model.Model, ")") + for _, comp := range comps { + comp.Version = defVersion + if comp.Metadata == nil { + comp.Metadata = make(map[string]interface{}) + } + // Assign the component status corresponding to model status. + // i.e. If model is enabled comps are also "enabled". Ultimately all individual comps itself will have ability to control their status. + // The status "enabled" indicates that the component will be registered inside the registry. + comp.Model = *modelDef + assignDefaultsForCompDefs(&comp, modelDef) + err := comp.WriteComponentDefinition(compDirPath) + if err != nil { + Log.Info(err) + } + } + + spreadsheeetChan <- registry.SpreadsheetData{ + Model: &model, + Components: comps, + } + + modelToCompGenerateTracker.Set(model.Model, registry.CompGenerateTracker{ + TotalComps: len(comps), + Version: version, + }) + }(model) + + } + wg.Wait() + close(spreadsheeetChan) + wgForSpreadsheetUpdate.Wait() + return nil +} + +func writeModelDefToFileSystem(model *registry.ModelCSV, version, modelDefPath string) (*v1beta1.Model, error) { + modelDef := model.CreateModelDefinition(version, defVersion) + err := modelDef.WriteModelDefinition(modelDefPath+"/model.json", "json") + if err != nil { + return nil, err + } + + return &modelDef, nil +} + +func rateLimitArtifactHub() { + artifactHubMutex.Lock() + defer artifactHubMutex.Unlock() + + if artifactHubCount > 0 && artifactHubCount%artifactHubRateLimit == 0 { + Log.Info("Rate limit reached for Artifact Hub. Sleeping for 5 minutes...") + time.Sleep(artifactHubRateLimitDur) + } + artifactHubCount++ +} + +func logModelGenerationSummary(modelToCompGenerateTracker *store.GenerticThreadSafeStore[registry.CompGenerateTracker]) { + for key, val := range modelToCompGenerateTracker.GetAllPairs() { + Log.Info(fmt.Sprintf("Generated %d components for model [%s] %s", val.TotalComps, key, val.Version)) + totalAggregateComponents += val.TotalComps + totalAggregateModel++ + } + + Log.Info(fmt.Sprintf("-----------------------------\n-----------------------------\nGenerated %d models and %d components", totalAggregateModel, totalAggregateComponents)) +} + +func parseModelSheet(url string) (*registry.ModelCSVHelper, error) { + modelCSVHelper, err := registry.NewModelCSVHelper(url, "Models", sheetGID) + if err != nil { + return nil, err + } + + err = modelCSVHelper.ParseModelsSheet(false) + if err != nil { + return nil, registry.ErrGenerateModel(err, "unable to start model generation") + } + return modelCSVHelper, nil +} + +func parseComponentSheet(url string) (*registry.ComponentCSVHelper, error) { + compCSVHelper, err := registry.NewComponentCSVHelper(url, "Components", componentSpredsheetGID) + if err != nil { + return nil, err + } + err = compCSVHelper.ParseComponentsSheet() + if err != nil { + return nil, registry.ErrGenerateModel(err, "unable to start model generation") + } + return compCSVHelper, nil +} + +func CreateVersionedDirectoryForModelAndComp(version, modelName string) (string, string, error) { + modelDirPath := filepath.Join(registryLocation, modelName, version, defVersion) + err := utils.CreateDirectory(modelDirPath) + if err != nil { + return "", "", err + } + + compDirPath := filepath.Join(modelDirPath, "components") + err = utils.CreateDirectory(compDirPath) + return modelDirPath, compDirPath, err +} + + +func GenerateDefsForCoreRegistrant(model registry.ModelCSV) error { + totalComps := 0 + var version string + defer func() { + modelToCompGenerateTracker.Set(model.Model, registry.CompGenerateTracker{ + TotalComps: totalComps, + Version: version, + }) + }() + + path, err := url.Parse(model.SourceURL) + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + LogError.Error(err) + return nil + } + gitRepo := github.GitRepo{ + URL: path, + PackageName: model.Model, + } + owner, repo, branch, root, err := gitRepo.ExtractRepoDetailsFromSourceURL() + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + LogError.Error(err) + return nil + } + + isModelPublished, _ := strconv.ParseBool(model.PublishToRegistry) + //Initialize walker + gitWalker := walker.NewGit() + if isModelPublished { + gw := gitWalker. + Owner(owner). + Repo(repo). + Branch(branch). + Root(root). + RegisterFileInterceptor(func(f walker.File) error { + // Check if the file has a JSON extension + if filepath.Ext(f.Name) != ".json" { + return nil + } + contentBytes := []byte(f.Content) + var componentDef v1beta1.ComponentDefinition + if err := json.Unmarshal(contentBytes, &componentDef); err != nil { + return err + } + version = componentDef.Model.Model.Version + modelDirPath, compDirPath, err := CreateVersionedDirectoryForModelAndComp(version, model.Model) + if err != nil { + err = registry.ErrGenerateModel(err, model.Model) + return err + } + _, err = writeModelDefToFileSystem(&model, version, modelDirPath) // how to infer this? @Beginner86 any idea? new column? + if err != nil { + return registry.ErrGenerateModel(err, model.Model) + } + + err = componentDef.WriteComponentDefinition(compDirPath) + if err != nil { + err = utils.ErrComponentGenerate(err) + LogError.Error(err) + } + return nil + }) + err = gw.Walk() + if err != nil { + return err + } + } + + return nil +} + +func assignDefaultsForCompDefs(componentDef *v1beta1.ComponentDefinition, modelDef *v1beta1.Model) { + componentDef.Metadata["status"] = modelDef.Status + for k, v := range modelDef.Metadata { + componentDef.Metadata[k] = v + } +} diff --git a/go.mod b/go.mod index d413cafa..4d7f0f6f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/fluxcd/pkg/tar v0.4.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-logr/logr v1.3.0 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/gofrs/uuid v4.4.0+incompatible github.com/google/go-containerregistry v0.17.0 github.com/google/uuid v1.5.0 @@ -33,6 +34,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.17.0 golang.org/x/oauth2 v0.15.0 + golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 google.golang.org/api v0.152.0 gopkg.in/yaml.v2 v2.4.0 @@ -237,7 +239,6 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index d2feebea..479ffbba 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,8 @@ github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XE github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= diff --git a/helpers/component_info.json b/helpers/component_info.json index 92af709d..3fb70b72 100644 --- a/helpers/component_info.json +++ b/helpers/component_info.json @@ -1,5 +1,5 @@ { "name": "meshkit", "type": "library", - "next_error_code": 11245 + "next_error_code": 11264 } \ No newline at end of file diff --git a/models/meshmodel/registry/error.go b/models/meshmodel/registry/error.go index 2566ec58..7274ddf7 100644 --- a/models/meshmodel/registry/error.go +++ b/models/meshmodel/registry/error.go @@ -7,14 +7,14 @@ import ( ) var ( - ErrUnknownHostCode = "replace_me" - ErrEmptySchemaCode = "replace_me" - ErrMarshalingRegisteryAttemptsCode = "replace_me" - ErrWritingRegisteryAttemptsCode = "replace_me" - ErrRegisteringEntityCode = "replace_me" - ErrUnknownHostInMapCode = "replace_me" - ErrCreatingUserDataDirectoryCode = "replace_me" - ErrGetByIdCode = "replace_me" + ErrUnknownHostCode = "meshkit-11247" + ErrEmptySchemaCode = "meshkit-11248" + ErrMarshalingRegisteryAttemptsCode = "meshkit-11249" + ErrWritingRegisteryAttemptsCode = "meshkit-11250" + ErrRegisteringEntityCode = "meshkit-11251" + ErrUnknownHostInMapCode = "meshkit-11252" + ErrCreatingUserDataDirectoryCode = "meshkit-11253" + ErrGetByIdCode = "meshkit-11254" ) func ErrGetById(err error, id string) error { diff --git a/models/patterns/error.go b/models/patterns/error.go index ad37aad6..03a95ab7 100644 --- a/models/patterns/error.go +++ b/models/patterns/error.go @@ -3,7 +3,7 @@ package patterns import "github.com/layer5io/meshkit/errors" const ( - ErrInvalidVersionCode = "" + ErrInvalidVersionCode = "meshkit-11254" ) func ErrInvalidVersion(err error) error { diff --git a/utils/error.go b/utils/error.go index 8e9a42cf..1fae0700 100644 --- a/utils/error.go +++ b/utils/error.go @@ -40,12 +40,13 @@ var ( ErrExtractTarXZCode = "meshkit-11184" ErrExtractZipCode = "meshkit-11185" ErrReadDirCode = "meshkit-11186" - ErrInvalidSchemaVersionCode = "replace_me" - ErrFileWalkDirCode = "replace_me" - ErrRelPathCode = "replace_me" - ErrCopyFileCode = "replace_me" - ErrCloseFileCode = "replace_me" + ErrInvalidSchemaVersionCode = "meshkit-11255" + ErrFileWalkDirCode = "meshkit-11256" + ErrRelPathCode = "meshkit-11257" + ErrCopyFileCode = "meshkit-11258" + ErrCloseFileCode = "meshkit-11259" ErrCompressToTarGZCode = "meshkit-11248" + ErrComponentGenerateCode = "meshkit-11260" ) var ( ErrExtractType = errors.New( @@ -66,6 +67,10 @@ var ( ) ) +func ErrComponentGenerate(err error) error { + return errors.New(ErrComponentGenerateCode, errors.Alert, []string{"failed to generate components for the package"}, []string{err.Error()}, []string{}, []string{"Make sure that the package is compatible"}) +} + func ErrCueLookup(err error) error { return errors.New(ErrCueLookupCode, errors.Alert, []string{"Could not lookup the given path in the CUE value"}, []string{err.Error()}, []string{""}, []string{"make sure that the path is a valid cue expression and is correct", "make sure that there exists a field with the given path", "make sure that the given root value is correct"}) } diff --git a/utils/registry/component.go b/utils/registry/component.go new file mode 100644 index 00000000..bb7c22a9 --- /dev/null +++ b/utils/registry/component.go @@ -0,0 +1,323 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/layer5io/meshkit/logger" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/csv" + "github.com/layer5io/meshkit/utils/manifests" +) + +const ( + SVG_WIDTH = 20 + SVG_HEIGHT = 20 + rowIndex = 1 + shouldRegisterColIndex = -1 +) + +var ( + Log logger.Handler + LogError logger.Handler +) + +type ComponentCSV struct { + Registrant string `json:"registrant" csv:"registrant"` + Model string `json:"model" csv:"model"` + Component string `json:"component" csv:"component"` + Description string `json:"description" csv:"description"` + Shape string `json:"shape" csv:"shape"` + PrimaryColor string `json:"primaryColor" csv:"primaryColor"` + SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` + SVGColor string `json:"svgColor" csv:"svgColor"` + SVGWhite string `json:"svgWhite" csv:"svgWhite"` + SVGComplete string `json:"svgComplete" csv:"svgComplete"` + HasSchema string `json:"hasSchema" csv:"hasSchema"` + Docs string `json:"docs" csv:"docs"` + StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` + Styles string `json:"styles" csv:"styles"` + ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` + DefaultData string `json:"defaultData" csv:"defaultData"` + Capabilities string `json:"capabilities" csv:"capabilities"` + LogoURL string `json:"logoURL" csv:"logoURL"` + Genealogy string `json:"genealogy" csv:"genealogy"` + IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` + + ModelDisplayName string `json:"modelDisplayName" csv:"-"` + Category string `json:"category" csv:"-"` + SubCategory string `json:"subCategory" csv:"-"` +} + +// The Component Definition generated assumes or is only for components which have registrant as "meshery" +func (c *ComponentCSV) CreateComponentDefinition(isModelPublished bool, defVersion string) (v1beta1.ComponentDefinition, error) { + componentDefinition := &v1beta1.ComponentDefinition{ + VersionMeta: v1beta1.VersionMeta{ + SchemaVersion: v1beta1.ComponentSchemaVersion, + Version: defVersion, + }, + DisplayName: c.Component, + Format: "JSON", + Metadata: map[string]interface{}{ + "published": isModelPublished, + }, + Component: v1beta1.ComponentEntity{}, + } + err := c.UpdateCompDefinition(componentDefinition) + return *componentDefinition, err +} + +var compMetadataValues = []string{ + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "genealogy", "isAnnotation", "shape", "subCategory", +} + +func (c *ComponentCSV) UpdateCompDefinition(compDef *v1beta1.ComponentDefinition) error { + + metadata := map[string]interface{}{} + compMetadata, err := utils.MarshalAndUnmarshal[ComponentCSV, map[string]interface{}](*c) + if err != nil { + return err + } + metadata = utils.MergeMaps(metadata, compDef.Metadata) + + for _, key := range compMetadataValues { + if key == "svgColor" || key == "svgWhite" { + svg, err := utils.Cast[string](compMetadata[key]) + if err == nil { + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT, false) + if err != nil { + // If svg cannot be updated, assign the svg value as it is + metadata[key] = compMetadata[key] + } + } + } + metadata[key] = compMetadata[key] + } + + isAnnotation := false + if strings.ToLower(c.IsAnnotation) == "true" { + isAnnotation = true + } + metadata["isAnnotation"] = isAnnotation + compDef.Metadata = metadata + return nil +} + +type ComponentCSVHelper struct { + SpreadsheetID int64 + SpreadsheetURL string + Title string + CSVPath string + Components map[string]map[string][]ComponentCSV +} + +func NewComponentCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ComponentCSVHelper, error) { + sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) + Log.Info("Downloading CSV from: ", sheetURL) + dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") + _ = os.MkdirAll(dirPath, 0755) + csvPath := filepath.Join(dirPath, "components.csv") + err := utils.DownloadFile(csvPath, sheetURL) + if err != nil { + return nil, utils.ErrReadingRemoteFile(err) + } + + return &ComponentCSVHelper{ + SpreadsheetID: spreadsheetID, + SpreadsheetURL: sheetURL, + Title: spreadsheetName, + CSVPath: csvPath, + Components: make(map[string]map[string][]ComponentCSV), + }, nil +} + +func (mch *ComponentCSVHelper) GetColumns() ([]string, error) { + csvReader, err := csv.NewCSVParser[ComponentCSV](mch.CSVPath, rowIndex, nil, func(_ []string, _ []string) bool { + return true + }) + if err != nil { + return nil, err + } + + return csvReader.ExtractCols(rowIndex) +} + +func (mch *ComponentCSVHelper) ParseComponentsSheet() error { + ch := make(chan ComponentCSV, 1) + errorChan := make(chan error, 1) + csvReader, err := csv.NewCSVParser[ComponentCSV](mch.CSVPath, rowIndex, nil, func(_ []string, _ []string) bool { + return true + }) + + if err != nil { + return utils.ErrReadFile(err, mch.CSVPath) + } + + go func() { + Log.Info("Parsing Components...") + + err := csvReader.Parse(ch, errorChan) + if err != nil { + errorChan <- err + } + }() + + for { + select { + + case data := <-ch: + if mch.Components[data.Registrant] == nil { + mch.Components[data.Registrant] = make(map[string][]ComponentCSV, 0) + } + if mch.Components[data.Registrant][data.Model] == nil { + mch.Components[data.Registrant][data.Model] = make([]ComponentCSV, 0) + } + mch.Components[data.Registrant][data.Model] = append(mch.Components[data.Registrant][data.Model], data) + Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s] component [%s]", data.Registrant, data.Model, data.Component)) + case err := <-errorChan: + Log.Error(err) + + case <-csvReader.Context.Done(): + return nil + } + } +} + +func CreateComponentsMetadataAndCreateSVGsForMDXStyle(model ModelCSV, components []ComponentCSV, path, svgDir string) (string, error) { + err := os.MkdirAll(filepath.Join(path, svgDir), 0777) + if err != nil { + return "", err + } + componentMetadata := `[` + for idx, comp := range components { + componentTemplate := ` +{ +"name": "%s", +"colorIcon": "%s", +"whiteIcon": "%s", +"description": "%s", +}` + + // add comma if not last component + if idx != len(components)-1 { + componentTemplate += "," + } + + compName := utils.FormatName(manifests.FormatToReadableString(comp.Component)) + colorIconDir := filepath.Join(svgDir, compName, "icons", "color") + whiteIconDir := filepath.Join(svgDir, compName, "icons", "white") + + componentMetadata += fmt.Sprintf(componentTemplate, compName, fmt.Sprintf("%s/%s-color.svg", colorIconDir, compName), fmt.Sprintf("%s/%s-white.svg", whiteIconDir, compName), comp.Description) + + // create color svg dir + err = os.MkdirAll(filepath.Join(path, colorIconDir), 0777) + if err != nil { + return "", err + } + + // create white svg dir + err = os.MkdirAll(filepath.Join(path, whiteIconDir), 0777) + if err != nil { + return "", err + } + + colorSVG, whiteSVG := getSVGForComponent(model, comp) + err = utils.WriteToFile(filepath.Join(path, colorIconDir, compName+"-color.svg"), colorSVG) + if err != nil { + return "", err + } + err = utils.WriteToFile(filepath.Join(path, whiteIconDir, compName+"-white.svg"), whiteSVG) + if err != nil { + return "", err + } + } + + componentMetadata += `]` + + return componentMetadata, nil +} + +func CreateComponentsMetadataAndCreateSVGsForMDStyle(model ModelCSV, components []ComponentCSV, path, svgDir string) (string, error) { + err := os.MkdirAll(filepath.Join(path), 0777) + if err != nil { + return "", err + } + componentMetadata := "" + for _, comp := range components { + componentTemplate := ` +- name: %s + colorIcon: %s + whiteIcon: %s + description: %s` + + compName := utils.FormatName(manifests.FormatToReadableString(comp.Component)) + colorIconDir := filepath.Join(svgDir, compName, "icons", "color") + whiteIconDir := filepath.Join(svgDir, compName, "icons", "white") + + componentMetadata += fmt.Sprintf(componentTemplate, compName, fmt.Sprintf("%s/%s-color.svg", colorIconDir, compName), fmt.Sprintf("%s/%s-white.svg", whiteIconDir, compName), comp.Description) + + // create color svg dir + err = os.MkdirAll(filepath.Join(path, compName, "icons", "color"), 0777) + if err != nil { + return "", err + } + + // create white svg dir + err = os.MkdirAll(filepath.Join(path, compName, "icons", "white"), 0777) + if err != nil { + return "", err + } + + colorSVG, whiteSVG := getSVGForComponent(model, comp) + err = utils.WriteToFile(filepath.Join(path, compName, "icons", "color", compName+"-color.svg"), colorSVG) + if err != nil { + return "", err + } + err = utils.WriteToFile(filepath.Join(path, compName, "icons", "white", compName+"-white.svg"), whiteSVG) + if err != nil { + return "", err + } + } + + return componentMetadata, nil +} + +func (m ComponentCSVHelper) Cleanup() error { + // remove csv file + Log.Info("Removing CSV file: ", m.CSVPath) + err := os.Remove(m.CSVPath) + if err != nil { + return err + } + return nil +} + +func ConvertCompDefToCompCSV(modelcsv *ModelCSV, compDef v1beta1.ComponentDefinition) *ComponentCSV { + compCSV, _ := utils.MarshalAndUnmarshal[map[string]interface{}, ComponentCSV](compDef.Metadata) + compCSV.Registrant = modelcsv.Registrant + compCSV.Model = modelcsv.Model + compCSV.Component = compDef.Component.Kind + compCSV.ModelDisplayName = modelcsv.ModelDisplayName + compCSV.Category = modelcsv.Category + compCSV.SubCategory = modelcsv.SubCategory + + return &compCSV +} + +func getSVGForComponent(model ModelCSV, component ComponentCSV) (colorSVG string, whiteSVG string) { + colorSVG = component.SVGColor + whiteSVG = component.SVGWhite + + if colorSVG == "" { + colorSVG = model.SVGColor + } + + if whiteSVG == "" { + whiteSVG = model.SVGWhite + } + return +} diff --git a/utils/registry/error.go b/utils/registry/error.go new file mode 100644 index 00000000..ab5c9309 --- /dev/null +++ b/utils/registry/error.go @@ -0,0 +1,32 @@ +package registry + +import ( + "fmt" + "github.com/layer5io/meshkit/errors" +) + +var ( + ErrAppendToSheetCode = "meshkit-11261" + ErrGenerateModelCode = "meshkit-11262" + ErrMarshalStructToCSVCode = "meshkit-11263" +) + +func ErrGenerateModel(err error, modelName string) error { + return errors.New(ErrGenerateModelCode, errors.Alert, []string{fmt.Sprintf("error generating model: %s", modelName)}, []string{fmt.Sprintf("Error generating model: %s\n %s", modelName, err.Error())}, []string{"Registrant used for the model is not supported", "Verify the model's source URL.", "Failed to create a local directory in the filesystem for this model."}, []string{"Ensure that each kind of registrant used is a supported kind.", "Ensure correct model source URL is provided and properly formatted.", "Ensure sufficient permissions to allow creation of model directory."}) +} + +func ErrAppendToSheet(err error, id string) error { + return errors.New(ErrAppendToSheetCode, errors.Alert, + []string{fmt.Sprintf("Failed to append data into sheet %s", id)}, + []string{err.Error()}, + []string{"Error occurred while appending to the spreadsheet", "The credential might be incorrect/expired"}, + []string{"Ensure correct append range (A1 notation) is used", "Ensure correct credential is used"}) +} + +func ErrMarshalStructToCSV(err error) error { + return errors.New(ErrMarshalStructToCSVCode, errors.Alert, + []string{"Failed to marshal struct to csv"}, + []string{err.Error()}, + []string{"The column names in your spreadsheet do not match the names in the struct.", " For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}, + []string{"The column names in the spreadsheet do not match the names in the struct. Please make sure they are spelled exactly the same and use the same case (uppercase/lowercase).", "The value you are trying to convert is not of the expected type for the column. Please ensure it is a [number, string, date, etc.].", "The column names in your spreadsheet do not match the names in the struct. For example, the spreadsheet has a column named 'First Name' but the struct expects a column named 'firstname'. Please make sure the names match exactly."}) +} diff --git a/utils/registry/model.go b/utils/registry/model.go new file mode 100644 index 00000000..681e72ba --- /dev/null +++ b/utils/registry/model.go @@ -0,0 +1,354 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/csv" +) + +var ( + shouldRegisterMod = "publishToSites" +) + +type ModelCSV struct { + Registrant string `json:"registrant" csv:"registrant"` + ModelDisplayName string `json:"modelDisplayName" csv:"modelDisplayName"` + Model string `json:"model" csv:"model"` + Category string `json:"category" csv:"category"` + SubCategory string `json:"subCategory" csv:"subCategory"` + Description string `json:"description" csv:"description"` + SourceURL string `json:"sourceURL" csv:"sourceURL"` + Website string `json:"website" csv:"website"` + Docs string `json:"docs" csv:"docs"` + Shape string `json:"shape" csv:"shape"` + PrimaryColor string `json:"primaryColor" csv:"primaryColor"` + SecondaryColor string `json:"secondaryColor" csv:"secondaryColor"` + StyleOverrides string `json:"styleOverrides" csv:"styleOverrides"` + Styles string `json:"styles" csv:"styles"` + ShapePolygonPoints string `json:"shapePolygonPoints" csv:"shapePolygonPoints"` + DefaultData string `json:"defaultData" csv:"defaultData"` + Capabilities string `json:"capabilities" csv:"capabilities"` + LogoURL string `json:"logoURL" csv:"logoURL"` + SVGColor string `json:"svgColor" csv:"svgColor"` + SVGWhite string `json:"svgWhite" csv:"svgWhite"` + SVGComplete string `json:"svgComplete" csv:"svgComplete"` + IsAnnotation string `json:"isAnnotation" csv:"isAnnotation"` + PublishToRegistry string `json:"publishToRegistry" csv:"publishToRegistry"` + AboutProject string `json:"aboutProject" csv:"-"` + PageSubtTitle string `json:"pageSubtitle" csv:"-"` + DocsURL string `json:"docsURL" csv:"-"` + StandardBlurb string `json:"standardBlurb" csv:"-"` + Feature1 string `json:"feature1" csv:"-"` + Feature2 string `json:"feature2" csv:"-"` + Feature3 string `json:"feature3" csv:"-"` + HowItWorks string `json:"howItWorks" csv:"-"` + HowItWorksDetails string `json:"howItWorksDetails" csv:"-"` + Screenshots string `json:"screenshots" csv:"-"` + FullPage string `json:"fullPage" csv:"-"` + PublishToSites string `json:"publishToSites" csv:"-"` +} + +var modelMetadataValues = []string{ + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "styles", "shapePolygonPoints", "defaultData", "capabilities", "isAnnotation", "shape", +} + +func (m *ModelCSV) UpdateModelDefinition(modelDef *v1beta1.Model) error { + + metadata := map[string]interface{}{} + modelMetadata, err := utils.MarshalAndUnmarshal[ModelCSV, map[string]interface{}](*m) + if err != nil { + return err + } + metadata = utils.MergeMaps(metadata, modelDef.Metadata) + + for _, key := range modelMetadataValues { + if key == "svgColor" || key == "svgWhite" { + svg, err := utils.Cast[string](modelMetadata[key]) + if err == nil { + metadata[key], err = utils.UpdateSVGString(svg, SVG_WIDTH, SVG_HEIGHT, false) + if err != nil { + // If svg cannot be updated, assign the svg value as it is + metadata[key] = modelMetadata[key] + } + } + } + metadata[key] = modelMetadata[key] + } + + isAnnotation := false + if strings.ToLower(m.IsAnnotation) == "true" { + isAnnotation = true + } + metadata["isAnnotation"] = isAnnotation + modelDef.Metadata = metadata + return nil +} + +func (mcv *ModelCSV) CreateModelDefinition(version, defVersion string) v1beta1.Model { + status := entity.Ignored + if strings.ToLower(mcv.PublishToRegistry) == "true" { + status = entity.Enabled + } + + model := v1beta1.Model{ + VersionMeta: v1beta1.VersionMeta{ + Version: defVersion, + SchemaVersion: v1beta1.ModelSchemaVersion, + }, + Name: mcv.Model, + DisplayName: mcv.ModelDisplayName, + Status: status, + Registrant: v1beta1.Host{ + Hostname: utils.ReplaceSpacesAndConvertToLowercase(mcv.Registrant), + }, + Category: v1beta1.Category{ + Name: mcv.Category, + }, + SubCategory: mcv.SubCategory, + Model: v1beta1.ModelEntity{ + Version: version, + }, + } + err := mcv.UpdateModelDefinition(&model) + if err != nil { + Log.Error(err) + } + return model +} + +type ModelCSVHelper struct { + SpreadsheetID int64 + SpreadsheetURL string + Title string + CSVPath string + Models []ModelCSV +} + +func NewModelCSVHelper(sheetURL, spreadsheetName string, spreadsheetID int64) (*ModelCSVHelper, error) { + sheetURL = sheetURL + "/pub?output=csv" + "&gid=" + strconv.FormatInt(spreadsheetID, 10) + Log.Info("Downloading CSV from: ", sheetURL) + dirPath := filepath.Join(utils.GetHome(), ".meshery", "content") + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return nil, utils.ErrCreateDir(err, dirPath) + } + csvPath := filepath.Join(dirPath, "models.csv") + err = utils.DownloadFile(csvPath, sheetURL) + if err != nil { + return nil, utils.ErrReadingRemoteFile(err) + } + + return &ModelCSVHelper{ + SpreadsheetID: spreadsheetID, + SpreadsheetURL: sheetURL, + Models: []ModelCSV{}, + CSVPath: csvPath, + Title: spreadsheetName, + }, nil +} + +func (mch *ModelCSVHelper) ParseModelsSheet(parseForDocs bool) error { + ch := make(chan ModelCSV, 1) + errorChan := make(chan error, 1) + csvReader, err := csv.NewCSVParser[ModelCSV](mch.CSVPath, rowIndex, nil, func(columns []string, currentRow []string) bool { + index := 0 + + if parseForDocs { + index = GetIndexForRegisterCol(columns, shouldRegisterMod) + } else { + // Generation of models should not consider publishedToRegistry column value. + // Generation should happen for all models, while during registration "published" attribute should be respected. + return true + } + if index != -1 && index < len(currentRow) { + shouldRegister := currentRow[index] + return strings.ToLower(shouldRegister) == "true" + } + return false + }) + + if err != nil { + return utils.ErrReadFile(err, mch.CSVPath) + } + + go func() { + Log.Info("Parsing Models...") + err := csvReader.Parse(ch, errorChan) + if err != nil { + errorChan <- err + } + }() + for { + select { + + case data := <-ch: + mch.Models = append(mch.Models, data) + Log.Info(fmt.Sprintf("Reading registrant [%s] model [%s]", data.Registrant, data.Model)) + case err := <-errorChan: + return utils.ErrReadFile(err, mch.CSVPath) + + case <-csvReader.Context.Done(): + return nil + } + } +} + +func GetIndexForRegisterCol(cols []string, shouldRegister string) int { + if shouldRegisterColIndex != -1 { + return shouldRegisterColIndex + } + + for index, col := range cols { + if col == shouldRegister { + return index + } + } + return shouldRegisterColIndex +} + +func (m ModelCSV) CreateMarkDownForMDXStyle(componentsMetadata string) string { + formattedName := utils.FormatName(m.Model) + var template string = `--- +title: %s +subtitle: %s +integrationIcon: icons/color/%s-color.svg +darkModeIntegrationIcon: icons/white/%s-white.svg +docURL: %s +description: %s +category: %s +subcategory: %s +registrant: %s +components: %v +featureList: [ + "%s", + "%s", + "%s" +] +workingSlides: [ + %s, + %s +] +howItWorks: "%s" +howItWorksDetails: "%s" +published: %s +--- +

+%s +

+%s +` + markdown := fmt.Sprintf(template, + m.ModelDisplayName, + m.PageSubtTitle, + formattedName, + formattedName, + m.DocsURL, + m.Description, + m.Category, + m.SubCategory, + m.Registrant, + componentsMetadata, + m.Feature1, + m.Feature2, + m.Feature3, + `../_images/meshmap-visualizer.png`, + `../_images/meshmap-designer.png`, + m.HowItWorks, + m.HowItWorksDetails, + m.PublishToSites, + m.AboutProject, + m.StandardBlurb, + ) + markdown = strings.ReplaceAll(markdown, "\r", "\n") + return markdown +} + +// Creates JSON formatted meshmodel attribute item for JSON Style docs +func (m ModelCSV) CreateJSONItem(iconDir string) string { + formattedModelName := utils.FormatName(m.Model) + json := "{" + json += fmt.Sprintf("\"name\":\"%s\"", m.Model) + // If SVGs exist, then add the paths to json + if m.SVGColor != "" { + json += fmt.Sprintf(",\"color\":\"%s/icons/color/%s-color.svg\"", iconDir, formattedModelName) + } + + if m.SVGWhite != "" { + json += fmt.Sprintf(",\"white\":\"%s/icons/white/%s-white.svg\"", iconDir, formattedModelName) + } + + json += fmt.Sprintf(",\"permalink\":\"%s\"", m.DocsURL) + + json += "}" + return json +} + +func (m ModelCSV) CreateMarkDownForMDStyle(componentsMetadata string) string { + formattedName := utils.FormatName(m.Model) + + var template string = `--- +layout: integration +title: %s +subtitle: %s +image: /assets/img/integrations/%s/icons/color/%s-color.svg +permalink: extensibility/integrations/%s +docURL: %s +description: %s +integrations-category: %s +integrations-subcategory: %s +registrant: %s +components: %v +featureList: [ + "%s", + "%s", + "%s" +] +howItWorks: "%s" +howItWorksDetails: "%s" +language: en +list: include +type: extensibility +category: integrations +--- +` + markdown := fmt.Sprintf(template, + m.ModelDisplayName, + m.PageSubtTitle, + formattedName, + formattedName, + formattedName, + m.DocsURL, + m.Description, + m.Category, + m.SubCategory, + m.Registrant, + componentsMetadata, + m.Feature1, + m.Feature2, + m.Feature3, + m.HowItWorks, + m.HowItWorksDetails, + ) + + markdown = strings.ReplaceAll(markdown, "\r", "\n") + + return markdown +} + +func (m ModelCSVHelper) Cleanup() error { + // remove csv file + Log.Info("Removing CSV file: ", m.CSVPath) + err := os.Remove(m.CSVPath) + if err != nil { + return err + } + + return nil +} diff --git a/utils/registry/spreadsheet_update.go b/utils/registry/spreadsheet_update.go new file mode 100644 index 00000000..52cae39c --- /dev/null +++ b/utils/registry/spreadsheet_update.go @@ -0,0 +1,233 @@ +package registry + +import ( + "bytes" + "context" + "sync" + + cuecsv "cuelang.org/go/pkg/encoding/csv" + "github.com/gocarina/gocsv" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "google.golang.org/api/sheets/v4" +) + +var ( + compBatchSize = 100 + modelBatchSize = 100 + + // refers to all cells in the fourth row (explain how is table clulte in ghsset and henc this rang is valid and will not reuire udpate or when it hould require update) + ComponentsSheetAppendRange = "Components!A4" + ModelsSheetAppendRange = "Models!A4" +) + +// registrant:model:component:[true/false] +// Tracks if component sheet requires update +var RegistrantToModelsToComponentsMap = make(map[string]map[string]map[string]bool) + +// registrant:model:[true/false] +// Tracks if component sheet requires update +var RegistrantToModelsMap = make(map[string]map[string]bool) + +func ProcessModelToComponentsMap(existingComponents map[string]map[string][]ComponentCSV) { + RegistrantToModelsToComponentsMap = make(map[string]map[string]map[string]bool, len(existingComponents)) + for registrant, models := range existingComponents { + for model, comps := range models { + if RegistrantToModelsToComponentsMap[registrant] == nil { + RegistrantToModelsToComponentsMap[registrant] = make(map[string]map[string]bool) + } + for _, comp := range comps { + if RegistrantToModelsToComponentsMap[registrant][model] == nil { + RegistrantToModelsToComponentsMap[registrant][model] = make(map[string]bool) + } + RegistrantToModelsToComponentsMap[registrant][model][comp.Component] = true + } + } + } +} + +func addEntriesInCompUpdateList(modelEntry *ModelCSV, compEntries []v1beta1.ComponentDefinition, compList []*ComponentCSV) []*ComponentCSV { + registrant := modelEntry.Registrant + model := modelEntry.Model + + if RegistrantToModelsToComponentsMap[registrant][model] == nil { + RegistrantToModelsToComponentsMap[registrant][model] = make(map[string]bool) + } + + for _, comp := range compEntries { + if !RegistrantToModelsToComponentsMap[registrant][model][comp.Component.Kind] { + RegistrantToModelsToComponentsMap[registrant][model][comp.Component.Kind] = true + compList = append(compList, ConvertCompDefToCompCSV(modelEntry, comp)) + compBatchSize-- + } + } + + return compList +} + +func addEntriesInModelUpdateList(modelEntry *ModelCSV, modelList []*ModelCSV) []*ModelCSV { + registrant := modelEntry.Registrant + + if RegistrantToModelsMap[registrant] == nil { + RegistrantToModelsMap[registrant] = make(map[string]bool) + } + RegistrantToModelsMap[registrant][modelEntry.Model] = true + modelBatchSize-- + + return modelList +} + +// Verifies if the component entry already exist in the spreadsheet, otherwise updates the spreadshhet to include new component entry. +func VerifyandUpdateSpreadsheet(cred string, wg *sync.WaitGroup, srv *sheets.Service, spreadsheetUpdateChan chan SpreadsheetData, sheetId string) { + defer wg.Done() + + entriesToBeAddedInCompSheet := []*ComponentCSV{} + entriesToBeAddedInModelSheet := []*ModelCSV{} + + for data := range spreadsheetUpdateChan { + _, ok := RegistrantToModelsMap[data.Model.Registrant] + if !ok { + entriesToBeAddedInModelSheet = addEntriesInModelUpdateList(data.Model, entriesToBeAddedInModelSheet) + } + + for _, comp := range data.Components { + existingModels, ok := RegistrantToModelsToComponentsMap[data.Model.Registrant] // replace with registrantr + if ok { + + existingComps, ok := existingModels[data.Model.Model] + + if ok { + entryExist := existingComps[comp.Component.Kind] + + if !entryExist { + entriesToBeAddedInCompSheet = append(entriesToBeAddedInCompSheet, ConvertCompDefToCompCSV(data.Model, comp)) + compBatchSize-- + RegistrantToModelsToComponentsMap[data.Model.Registrant][data.Model.Model][comp.Component.Kind] = true + } + } else { + entriesToBeAddedInCompSheet = addEntriesInCompUpdateList(data.Model, data.Components, entriesToBeAddedInCompSheet) + } + } else { + + RegistrantToModelsToComponentsMap[data.Model.Registrant] = make(map[string]map[string]bool) + entriesToBeAddedInCompSheet = addEntriesInCompUpdateList(data.Model, data.Components, entriesToBeAddedInCompSheet) + } + } + + if modelBatchSize <= 0 { + // update model spreadsheet + err := updateModelsSheet(srv, cred, sheetId, entriesToBeAddedInModelSheet) + // Reset the list + entriesToBeAddedInModelSheet = []*ModelCSV{} + if err != nil { + Log.Error(err) + } + } + + if compBatchSize <= 0 { + // update comp spreadsheet + err := updateComponentsSheet(srv, cred, sheetId, entriesToBeAddedInCompSheet) + // Reset the list + entriesToBeAddedInCompSheet = []*ComponentCSV{} + entriesToBeAddedInModelSheet = []*ModelCSV{} + if err != nil { + Log.Error(err) + } + } + } + + if len(entriesToBeAddedInModelSheet) > 0 { + err := updateModelsSheet(srv, cred, sheetId, entriesToBeAddedInModelSheet) + if err != nil { + Log.Error(err) + } + } + + if len(entriesToBeAddedInCompSheet) > 0 { + err := updateComponentsSheet(srv, cred, sheetId, entriesToBeAddedInCompSheet) + if err != nil { + Log.Error(err) + } + return + } +} + +func updateModelsSheet(srv *sheets.Service, cred, sheetId string, values []*ModelCSV) error { + marshalledValues, err := marshalStructToCSValues[ModelCSV](values) + if err != nil { + return err + } + Log.Info("Appending", len(marshalledValues), "in the models sheet") + err = appendSheet(srv, cred, sheetId, ModelsSheetAppendRange, marshalledValues) + + return err +} + +func updateComponentsSheet(srv *sheets.Service, cred, sheetId string, values []*ComponentCSV) error { + marshalledValues, err := marshalStructToCSValues[ComponentCSV](values) + Log.Info("Appending", len(marshalledValues), "in the components sheet") + if err != nil { + return err + } + err = appendSheet(srv, cred, sheetId, ComponentsSheetAppendRange, marshalledValues) + + return err +} + +func appendSheet(srv *sheets.Service, cred, sheetId, appendRange string, values [][]interface{}) error { + + if len(values) == 0 { + return nil + } + _, err := srv.Spreadsheets.Values.Append(sheetId, appendRange, &sheets.ValueRange{ + MajorDimension: "ROWS", + Range: appendRange, + Values: values, + }).InsertDataOption("INSERT_ROWS").ValueInputOption("USER_ENTERED").Context(context.Background()).Do() + + if err != nil { + return ErrAppendToSheet(err, sheetId) + } + return nil +} + +func marshalStructToCSValues[K any](data []*K) ([][]interface{}, error) { + csvString, err := gocsv.MarshalString(data) + if err != nil { + return nil, ErrMarshalStructToCSV(err) + } + csvReader := bytes.NewBufferString(csvString) + decodedCSV, err := cuecsv.Decode(csvReader) + + if err != nil { + return nil, ErrMarshalStructToCSV(err) + } + + results := make([][]interface{}, 0) + + // The ouput is [ [col-names...] [row1][row2]] + // delete the first entry i.e. [col-names..], as it contains the column names and is not required as we are concerened only with rows + if len(decodedCSV) > 0 { + for idx, val := range decodedCSV { + if idx == 0 { + continue + } + result := make([]interface{}, 0, cap(val)) + for _, r := range val { + result = append(result, r) + } + results = append(results, result) + } + return results, nil + } + + return results, nil +} + +func GetSheetIDFromTitle(s *sheets.Spreadsheet, title string) int64 { + for _, sheet := range s.Sheets { + if sheet.Properties.Title == title { + return sheet.Properties.SheetId + } + } + return -1 +} diff --git a/utils/registry/types.go b/utils/registry/types.go new file mode 100644 index 00000000..29d00a18 --- /dev/null +++ b/utils/registry/types.go @@ -0,0 +1,15 @@ +package registry + +import ( + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" +) + +type SpreadsheetData struct { + Model *ModelCSV + Components []v1beta1.ComponentDefinition +} + +type CompGenerateTracker struct { + TotalComps int + Version string +}