Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replace template parsing with yaml.Decoder #42

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Replace template parsing with yaml.Decoder #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b951fdd
Replace template parsing with yaml.Decoder
Aug 28, 2024
7d26eaf
Fix test error
Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.8.4
github.com/thoas/go-funk v0.9.3
gopkg.in/yaml.v3 v3.0.1
helm.sh/helm/v3 v3.14.2
k8s.io/api v0.29.0
)
Expand Down Expand Up @@ -149,7 +150,6 @@ require (
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.29.0 // indirect
k8s.io/apimachinery v0.29.0 // indirect
k8s.io/apiserver v0.29.0 // indirect
Expand Down
43 changes: 32 additions & 11 deletions pkg/images.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package pkg

import (
"bytes"
"errors"
"fmt"
"io"
"os"
"reflect"
"regexp"
"strings"

"github.com/nikhilsbhat/common/renderer"
Expand All @@ -13,6 +15,7 @@ import (
monitoringV1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
"github.com/sirupsen/logrus"
"github.com/thoas/go-funk"
"gopkg.in/yaml.v3"
)

const (
Expand Down Expand Up @@ -117,7 +120,11 @@ func (image *Images) GetImages() error {
}

images := make([]*k8s.Image, 0)
kubeKindTemplates := image.GetTemplates(chart)
kubeKindTemplates, err := image.GetTemplates(chart)
if err != nil {
return err
}

skips := image.GetResourcesToSkip()

for _, kubeKindTemplate := range kubeKindTemplates {
Expand Down Expand Up @@ -194,15 +201,29 @@ func (image *Images) getChartManifests() ([]byte, error) {
return image.getChartFromTemplate()
}

// GetTemplates returns the split manifests fetched from one big template string fetched from `helm template`.
func (image *Images) GetTemplates(template []byte) []string {
image.log.Debugf("splitting helm manifests with regex pattern: '%s'", image.ImageRegex)
temp := regexp.MustCompile(image.ImageRegex)
kinds := temp.Split(string(template), -1)
// Removing empty string at the beginning as splitting string always adds it in front.
kinds = kinds[1:]
// GetTemplates returns each of the manifests fetched from one big template string fetched from `helm template`.
func (image *Images) GetTemplates(template []byte) ([]map[string]interface{}, error) {
var templates []map[string]interface{}
decoder := yaml.NewDecoder(bytes.NewReader(template))
docCount := 0

for {
var doc map[string]interface{}
docCount += 1
err := decoder.Decode(&doc)
if err != nil {
if err == io.EOF {
break
}
return nil, fmt.Errorf("error decoding YAML in document %d: %v", docCount, err)
}

if len(doc) > 0 {
templates = append(templates, doc)
}
}

return kinds
return templates, nil
}

// GetResourcesToSkip returns the skip from translating the flags.
Expand All @@ -229,7 +250,7 @@ func (image *Images) GetResourcesToSkip() []Skip {
// GetImage returns []*k8s.Image from the kubernetes manifests.
//
//nolint:gocognit,funlen
func (image *Images) GetImage(currentKind, kubeKindTemplate string) ([]*k8s.Image, error) {
func (image *Images) GetImage(currentKind string, kubeKindTemplate map[string]interface{}) ([]*k8s.Image, error) {
images := make([]*k8s.Image, 0)

switch currentKind {
Expand Down
6 changes: 5 additions & 1 deletion pkg/images_all.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@ func (image *Images) GetAllImages() error {
image.log.Debugf("fetching the images from release '%s' of namespace '%s'", release.Name, release.Namespace)

images := make([]*k8s.Image, 0)
kubeKindTemplates := image.GetTemplates([]byte(release.Manifest))
kubeKindTemplates, err := image.GetTemplates([]byte(release.Manifest))
if err != nil {
return err
}

skips := image.GetResourcesToSkip()

for _, kubeKindTemplate := range kubeKindTemplates {
Expand Down
40 changes: 18 additions & 22 deletions pkg/images_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,11 @@ roleRef:
name: prometheus-standalone-kube-state-metrics
subjects:
- kind: ServiceAccount
name: prometheus-standalone-kube-state-metrics
namespace: test
name: prometheus-standalone-kube-state-metrics
namespace: test
---
# Empty template
# Just comments
---
# Source: tracing/templates/jaeger/configmap.yaml
apiVersion: v1
Expand All @@ -71,30 +74,23 @@ metadata:
data:
CA_CERTIFICATE: |
-----BEGIN CERTIFICATE-----
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
OCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$
-----END CERTIFICATE-----
`

t.Run("should be able to split rendered templates to individual templates", func(t *testing.T) {
expected := []string{
"\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n component: \"alertmanager\"\n app: prometheus\n release: prometheus-standalone\n chart: prometheus-14.4.1\n heritage: Helm\n name: prometheus-standalone-alertmanager\nrules:\n []\n", //nolint:lll
"\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n component: \"pushgateway\"\n app: prometheus\n release: prometheus-standalone\n chart: prometheus-14.4.1\n heritage: Helm\n name: prometheus-standalone-pushgateway\nrules:\n []\n", //nolint:lll
"\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n labels:\n app.kubernetes.io/name: kube-state-metrics\n helm.sh/chart: kube-state-metrics-3.1.1\n app.kubernetes.io/managed-by: Helm\n app.kubernetes.io/instance: prometheus-standalone\n name: prometheus-standalone-kube-state-metrics\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: prometheus-standalone-kube-state-metrics\nsubjects:\n- kind: ServiceAccount\n name: prometheus-standalone-kube-state-metrics\n namespace: test\n", //nolint:lll
"\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: jaeger-ca-cert\ndata:\n CA_CERTIFICATE: |\n -----BEGIN CERTIFICATE-----\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n\t\tOCOIRRGVEGHEIGHEnwoircne20394809234nfh834retitneh83t5ljfKHD&$&$\n -----END CERTIFICATE-----\n", //nolint:lll
}
actual := imageClient.GetTemplates([]byte(helmTemplate))
assert.Equal(t, len(expected), len(actual))
// assert.Equal(t, sort.StringSlice(expected), sort.StringSlice(actual))
assert.ElementsMatch(t, expected, actual)
actual, err := imageClient.GetTemplates([]byte(helmTemplate))
assert.NoError(t, err)
assert.Len(t, actual, 4)
})
}

Expand Down
100 changes: 18 additions & 82 deletions pkg/k8s/k8s.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ type (

// KindInterface implements method that identifies the type of kubernetes workloads.
type KindInterface interface {
Get(dataMap string, log *logrus.Logger) (string, error)
Get(dataMap map[string]interface{}, log *logrus.Logger) (string, error)
}

// ImagesInterface implements method that gets images from various kubernetes workloads.
type ImagesInterface interface {
Get(dataMap string, imageRegex string, log *logrus.Logger) (*Image, error)
Get(dataMap map[string]interface{}, imageRegex string, log *logrus.Logger) (*Image, error)
}

// Image holds information of images retrieved.
Expand All @@ -85,11 +85,7 @@ type Images struct {
NameSpace string `json:"name_space,omitempty" yaml:"name_space,omitempty"`
}

func (name *Name) Get(dataMap string, log *logrus.Logger) (string, error) {
if err := yaml.Unmarshal([]byte(dataMap), name); err != nil {
return "", err
}

func (name *Name) Get(dataMap map[string]interface{}, log *logrus.Logger) (string, error) {
kindYaml := *name

metadata, metadataExists := kindYaml["metadata"].(map[string]interface{})
Expand All @@ -107,11 +103,7 @@ func (name *Name) Get(dataMap string, log *logrus.Logger) (string, error) {
return metadataName, nil
}

func (kin *Kind) Get(dataMap string, log *logrus.Logger) (string, error) {
if err := yaml.Unmarshal([]byte(dataMap), &kin); err != nil {
return "", err
}

func (kin *Kind) Get(dataMap map[string]interface{}, log *logrus.Logger) (string, error) {
if kin == nil {
log.Warn("looks like it manifest is empty")

Expand All @@ -130,11 +122,7 @@ func (kin *Kind) Get(dataMap string, log *logrus.Logger) (string, error) {
}

// Get identifies images from Deployments.
func (dep *Deployments) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Deployments) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Template.Spec.Containers, dep.Spec.Template.Spec.InitContainers...)}

images := &Image{
Expand All @@ -149,11 +137,7 @@ func (dep *Deployments) Get(dataMap string, _ string, _ *logrus.Logger) (*Image,
}

// Get identifies images from StatefulSets.
func (dep *StatefulSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *StatefulSets) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Template.Spec.Containers, dep.Spec.Template.Spec.InitContainers...)}

images := &Image{
Expand All @@ -168,11 +152,7 @@ func (dep *StatefulSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image
}

// Get identifies images from DaemonSets.
func (dep *DaemonSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *DaemonSets) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Template.Spec.Containers, dep.Spec.Template.Spec.InitContainers...)}

images := &Image{
Expand All @@ -187,11 +167,7 @@ func (dep *DaemonSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image,
}

// Get identifies images from CronJob.
func (dep *CronJob) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *CronJob) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.JobTemplate.Spec.Template.Spec.Containers,
dep.Spec.JobTemplate.Spec.Template.Spec.InitContainers...)}

Expand All @@ -207,11 +183,7 @@ func (dep *CronJob) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, err
}

// Get identifies images from Job.
func (dep *Job) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Job) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Template.Spec.Containers, dep.Spec.Template.Spec.InitContainers...)}

images := &Image{
Expand All @@ -226,11 +198,7 @@ func (dep *Job) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error)
}

// Get identifies images from ReplicaSets.
func (dep *ReplicaSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *ReplicaSets) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Template.Spec.Containers, dep.Spec.Template.Spec.InitContainers...)}

images := &Image{
Expand All @@ -245,11 +213,7 @@ func (dep *ReplicaSets) Get(dataMap string, _ string, _ *logrus.Logger) (*Image,
}

// Get identifies images from Pod.
func (dep *Pod) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Pod) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
depContainers := containers{append(dep.Spec.Containers, dep.Spec.InitContainers...)}

images := &Image{
Expand All @@ -264,11 +228,7 @@ func (dep *Pod) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error)
}

// Get identifies images from AlertManager.
func (dep *AlertManager) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *AlertManager) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
images := &Image{
Kind: monitoringV1.AlertmanagersKind,
Name: dep.Name,
Expand All @@ -279,11 +239,7 @@ func (dep *AlertManager) Get(dataMap string, _ string, _ *logrus.Logger) (*Image
}

// Get identifies images from Prometheus.
func (dep *Prometheus) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Prometheus) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
var imageNames []string

depContainers := containers{append(dep.Spec.Containers, dep.Spec.InitContainers...)}
Expand All @@ -301,11 +257,7 @@ func (dep *Prometheus) Get(dataMap string, _ string, _ *logrus.Logger) (*Image,
}

// Get identifies images from ThanosRuler.
func (dep *ThanosRuler) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *ThanosRuler) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
var imageNames []string

depContainers := containers{append(dep.Spec.Containers, dep.Spec.InitContainers...)}
Expand All @@ -323,11 +275,7 @@ func (dep *ThanosRuler) Get(dataMap string, _ string, _ *logrus.Logger) (*Image,
}

// Get identifies images from Grafana.
func (dep *Grafana) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Grafana) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
if dep.APIVersion == "integreatly.org/v1alpha1" {
return nil, &imgErrors.GrafanaAPIVersionSupportError{
Message: fmt.Sprintf("plugin supports the latest api version and '%s' is not supported", dep.APIVersion),
Expand All @@ -347,11 +295,7 @@ func (dep *Grafana) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, err
}

// Get identifies images from Thanos.
func (dep *Thanos) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *Thanos) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
thanosContainers := make([]coreV1.Container, 0)
thanosContainers = append(thanosContainers, dep.Spec.Rule.StatefulsetOverrides.Spec.Template.Spec.Containers...)
thanosContainers = append(thanosContainers, dep.Spec.Rule.StatefulsetOverrides.Spec.Template.Spec.InitContainers...)
Expand All @@ -374,11 +318,7 @@ func (dep *Thanos) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, erro
}

// Get identifies images from ThanosReceiver.
func (dep *ThanosReceiver) Get(dataMap string, _ string, _ *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *ThanosReceiver) Get(dataMap map[string]interface{}, _ string, _ *logrus.Logger) (*Image, error) {
receiverGroupTotalContainers := make([]coreV1.Container, 0)

for _, receiverGroup := range dep.Spec.ReceiverGroups {
Expand All @@ -398,11 +338,7 @@ func (dep *ThanosReceiver) Get(dataMap string, _ string, _ *logrus.Logger) (*Ima
return images, nil
}

func (dep *ConfigMap) Get(dataMap string, imageRegex string, log *logrus.Logger) (*Image, error) {
if err := yaml.Unmarshal([]byte(dataMap), &dep); err != nil {
return nil, err
}

func (dep *ConfigMap) Get(dataMap map[string]interface{}, imageRegex string, log *logrus.Logger) (*Image, error) {
images := &Image{
Kind: KindConfigMap,
Name: dep.Name,
Expand Down
Loading