diff --git a/api/observability/v1/output_types.go b/api/observability/v1/output_types.go index 2d47a24e5..9e696c1c9 100644 --- a/api/observability/v1/output_types.go +++ b/api/observability/v1/output_types.go @@ -372,23 +372,50 @@ type Cloudwatch struct { // GroupName defines the strategy for grouping logstreams // - // The GroupName can be a combination of static and dynamic values consisting of field paths followed by "\|\|" followed by another field path or a static value. + // The GroupName can be a combination of static and dynamic values consisting of field paths followed by "||" followed by another field path or a static value. // - // A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "\|\|". + // A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "||". // // Static values can only contain alphanumeric characters along with dashes, underscores, dots and forward slashes. // // Example: // // 1. foo-{.bar\|\|"none"} - // // 2. {.foo\|\|.bar\|\|"missing"} - // // 3. foo.{.bar.baz\|\|.qux.quux.corge\|\|.grault\|\|"nil"}-waldo.fred{.plugh\|\|"none"} // // +kubebuilder:validation:Pattern:=`^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Group Name",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"} GroupName string `json:"groupName"` + + // GroupClass defines the log class to be used when creating a log group for the first time. + // Allowed values are "standard" and "infrequent_access" + // + // The log group class cannot be changed once the group is created. + // + // NOTE: The "delivery" log class is not supported due to its limited feature set and complex constraints + // + // +kubebuilder:validation:Optional + // +kubebuilder:default=standard + // +kubebuilder:validation:Enum=standard;infrequent_access;infrequentAccess + GroupClass string `json:"groupClass,omitempty"` + + // Tags is a map of key-value pairs that are applied to the CloudWatch log group. + // + ///* OPTIONALLY?? + // The values can be a combination of static and dynamic values, using the same + // templating syntax as GroupName. + // + // The key must be a valid AWS Tag key. + // The value must follow the same pattern as GroupName. + // NOTE: kubebuilder doesn't directly validate map values in this way + //*/ + // + // +kubebuilder:validation:Optional + // +nullable + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User-Defined Tags" + // +kubebuilder:validation:MaxProperties:=10 + Tags map[string]string `json:"tags,omitempty"` } // AwsAuthType sets the authentication type used for CloudWatch. @@ -1473,4 +1500,21 @@ type S3 struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Custom Endpoint URL",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"} URL string `json:"url,omitempty"` + + // Tags is a map of key-value pairs that are applied to the CloudWatch log group. + // + ///* OPTIONALLY?? + // The values can be a combination of static and dynamic values, using the same + // templating syntax as GroupName. + // + // The key must be a valid AWS Tag key. + // The value must follow the same pattern as GroupName. + // NOTE: kubebuilder doesn't directly validate map values in this way + //*/ + // + // +kubebuilder:validation:Optional + // +nullable + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="User-Defined Tags" + // +kubebuilder:validation:MaxProperties:=10 + Tags map[string]string `json:"tags,omitempty"` } diff --git a/api/observability/v1/zz_generated.deepcopy.go b/api/observability/v1/zz_generated.deepcopy.go index ae3d3f7bc..5cc1f3134 100644 --- a/api/observability/v1/zz_generated.deepcopy.go +++ b/api/observability/v1/zz_generated.deepcopy.go @@ -272,6 +272,13 @@ func (in *Cloudwatch) DeepCopyInto(out *Cloudwatch) { *out = new(CloudwatchTuningSpec) (*in).DeepCopyInto(*out) } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cloudwatch. diff --git a/bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml b/bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml index c5d6a9de8..b94bf3af4 100644 --- a/bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml +++ b/bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml @@ -2045,27 +2045,55 @@ spec: - message: Additional type specific spec is required for authentication rule: self.type != 'iamRole' || has(self.iamRole) + groupClass: + default: standard + description: |- + GroupClass defines the log class to be used when creating a log group for the first time. + Allowed values are "standard" and "infrequent_access" + + The log group class cannot be changed once the group is created. + + NOTE: The "delivery" log class is not supported due to its limited feature set and complex constraints + enum: + - standard + - infrequent_access + - infrequentAccess + type: string groupName: description: |- GroupName defines the strategy for grouping logstreams - The GroupName can be a combination of static and dynamic values consisting of field paths followed by "\|\|" followed by another field path or a static value. + The GroupName can be a combination of static and dynamic values consisting of field paths followed by "||" followed by another field path or a static value. - A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "\|\|". + A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "||". Static values can only contain alphanumeric characters along with dashes, underscores, dots and forward slashes. Example: 1. foo-{.bar\|\|"none"} - 2. {.foo\|\|.bar\|\|"missing"} - 3. foo.{.bar.baz\|\|.qux.quux.corge\|\|.grault\|\|"nil"}-waldo.fred{.plugh\|\|"none"} pattern: ^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$ type: string region: type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key-value pairs that are applied to the CloudWatch log group. + /* OPTIONALLY?? + The values can be a combination of static and dynamic values, using the same + templating syntax as GroupName. + + The map key must be a valid AWS Tag key. + The map value must follow the same pattern as GroupName. + NOTE: kubebuilder doesn't directly validate map values in this way + */ + maxProperties: 10 + nullable: true + type: object tuning: description: Tuning specs tuning for the output nullable: true diff --git a/config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml b/config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml index a57a0277a..abe694e66 100644 --- a/config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml +++ b/config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml @@ -2045,27 +2045,55 @@ spec: - message: Additional type specific spec is required for authentication rule: self.type != 'iamRole' || has(self.iamRole) + groupClass: + default: standard + description: |- + GroupClass defines the log class to be used when creating a log group for the first time. + Allowed values are "standard" and "infrequent_access" + + The log group class cannot be changed once the group is created. + + NOTE: The "delivery" log class is not supported due to its limited feature set and complex constraints + enum: + - standard + - infrequent_access + - infrequentAccess + type: string groupName: description: |- GroupName defines the strategy for grouping logstreams - The GroupName can be a combination of static and dynamic values consisting of field paths followed by "\|\|" followed by another field path or a static value. + The GroupName can be a combination of static and dynamic values consisting of field paths followed by "||" followed by another field path or a static value. - A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "\|\|". + A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "||". Static values can only contain alphanumeric characters along with dashes, underscores, dots and forward slashes. Example: 1. foo-{.bar\|\|"none"} - 2. {.foo\|\|.bar\|\|"missing"} - 3. foo.{.bar.baz\|\|.qux.quux.corge\|\|.grault\|\|"nil"}-waldo.fred{.plugh\|\|"none"} pattern: ^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$ type: string region: type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key-value pairs that are applied to the CloudWatch log group. + /* OPTIONALLY?? + The values can be a combination of static and dynamic values, using the same + templating syntax as GroupName. + + The map key must be a valid AWS Tag key. + The map value must follow the same pattern as GroupName. + NOTE: kubebuilder doesn't directly validate map values in this way + */ + maxProperties: 10 + nullable: true + type: object tuning: description: Tuning specs tuning for the output nullable: true diff --git a/config/manifests/bases/cluster-logging.clusterserviceversion.yaml b/config/manifests/bases/cluster-logging.clusterserviceversion.yaml index 516464397..1173fd14e 100644 --- a/config/manifests/bases/cluster-logging.clusterserviceversion.yaml +++ b/config/manifests/bases/cluster-logging.clusterserviceversion.yaml @@ -549,18 +549,16 @@ spec: - description: |- GroupName defines the strategy for grouping logstreams - The GroupName can be a combination of static and dynamic values consisting of field paths followed by "\|\|" followed by another field path or a static value. + The GroupName can be a combination of static and dynamic values consisting of field paths followed by "||" followed by another field path or a static value. - A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "\|\|". + A dynamic value is encased in single curly brackets "{}" and MUST end with a static fallback value separated with "||". Static values can only contain alphanumeric characters along with dashes, underscores, dots and forward slashes. Example: 1. foo-{.bar\|\|"none"} - 2. {.foo\|\|.bar\|\|"missing"} - 3. foo.{.bar.baz\|\|.qux.quux.corge\|\|.grault\|\|"nil"}-waldo.fred{.plugh\|\|"none"} displayName: Group Name path: outputs[0].cloudwatch.groupName @@ -570,6 +568,18 @@ spec: path: outputs[0].cloudwatch.region x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text + - description: |- + Tags is a map of key-value pairs that are applied to the CloudWatch log group. + /* OPTIONALLY?? + The values can be a combination of static and dynamic values, using the same + templating syntax as GroupName. + + The map key must be a valid AWS Tag key. + The map value must follow the same pattern as GroupName. + NOTE: kubebuilder doesn't directly validate map values in this way + */ + displayName: User-Defined Tags + path: outputs[0].cloudwatch.tags - description: Tuning specs tuning for the output displayName: Tuning Options path: outputs[0].cloudwatch.tuning diff --git a/internal/generator/vector/output/aws/cloudwatch/cloudwatch.go b/internal/generator/vector/output/aws/cloudwatch/cloudwatch.go index b1eb39c5b..e70b25a08 100644 --- a/internal/generator/vector/output/aws/cloudwatch/cloudwatch.go +++ b/internal/generator/vector/output/aws/cloudwatch/cloudwatch.go @@ -17,40 +17,30 @@ import ( commontemplate "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/common/template" ) -type Endpoint struct { - URL string -} - -func (e Endpoint) Name() string { - return "awsEndpointTemplate" -} - -func (e Endpoint) Template() (ret string) { - ret = `{{define "` + e.Name() + `" -}}` - if e.URL != "" { - ret += `endpoint = "{{ .URL }}"` - } - ret += `{{end}}` - return -} +const ( + StandardGroupClass = "standard" + InfrequentGroupClass = "infrequent_access" + InfrequentGroupClassLegacy = "infrequentAccess" +) type CloudWatch struct { - Desc string - ComponentID string - Inputs string - Region string - GroupName string - EndpointConfig Element - AuthConfig Element + Desc string + ComponentID string + Inputs string + Region string + GroupName string + GroupClassConfig Element + EndpointConfig Element + AuthConfig Element common.RootMixin } -func (e CloudWatch) Name() string { +func (c CloudWatch) Name() string { return "cloudwatchTemplate" } -func (e CloudWatch) Template() string { - return `{{define "` + e.Name() + `" -}} +func (c CloudWatch) Template() string { + return `{{define "` + c.Name() + `" -}} {{if .Desc -}} # {{.Desc}} {{end -}} @@ -60,6 +50,9 @@ inputs = {{.Inputs}} region = "{{.Region}}" {{.Compression}} group_name = "{{"{{"}} _internal.{{.GroupName}} {{"}}"}}" +{{compose_one .GroupClassConfig}} +# TESTING +# TESTING stream_name = "{{"{{ stream_name }}"}}" {{compose_one .AuthConfig}} healthcheck.enabled = false @@ -68,8 +61,42 @@ healthcheck.enabled = false ` } -func (e *CloudWatch) SetCompression(algo string) { - e.Compression.Value = algo +type LogGroupClass struct { + GroupClass string +} + +func (g LogGroupClass) Name() string { + return "awsLogGroupClassTemplate" +} + +func (g LogGroupClass) Template() (ret string) { + ret = `{{define "` + g.Name() + `" -}}` + if g.GroupClass != "" { + ret += `log_group_class = "{{ .GroupClass }}"` + } + ret += `{{end}}` + return +} + +type Endpoint struct { + URL string +} + +func (e Endpoint) Name() string { + return "awsEndpointTemplate" +} + +func (e Endpoint) Template() (ret string) { + ret = `{{define "` + e.Name() + `" -}}` + if e.URL != "" { + ret += `endpoint = "{{ .URL }}"` + } + ret += `{{end}}` + return +} + +func (c *CloudWatch) SetCompression(algo string) { + c.Compression.Value = algo } func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Secrets, strategy common.ConfigStrategy, op Options) []Element { @@ -81,6 +108,7 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec Debug(id, vectorhelpers.MakeInputs([]string{componentID}...)), } } + cwSink := sink(id, o, []string{groupNameID}, secrets, op, o.Cloudwatch.Region, groupNameID) if strategy != nil { strategy.VisitSink(cwSink) @@ -88,8 +116,9 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec return []Element{ NormalizeStreamName(componentID, inputs), - commontemplate.TemplateRemap(groupNameID, []string{componentID}, o.Cloudwatch.GroupName, groupNameID, "Cloudwatch Groupname"), + commontemplate.TemplateRemap(groupNameID, []string{componentID}, o.Cloudwatch.GroupName, groupNameID, "Cloudwatch GroupName"), cwSink, + aws.NewTags(id, o.Cloudwatch), common.NewEncoding(id, common.CodecJSON), common.NewAcknowledgments(id, strategy), common.NewBatch(id, strategy), @@ -101,14 +130,15 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec func sink(id string, o obs.OutputSpec, inputs []string, secrets observability.Secrets, op Options, region, groupName string) *CloudWatch { return &CloudWatch{ - Desc: "Cloudwatch Logs", - ComponentID: id, - Inputs: vectorhelpers.MakeInputs(inputs...), - Region: region, - GroupName: groupName, - AuthConfig: aws.AuthConfig(o.Name, o.Cloudwatch.Authentication, op, secrets), - EndpointConfig: endpointConfig(o.Cloudwatch), - RootMixin: common.NewRootMixin("none"), + Desc: "Cloudwatch Logs", + ComponentID: id, + Inputs: vectorhelpers.MakeInputs(inputs...), + Region: region, + GroupName: groupName, + GroupClassConfig: groupClassConfig(o.Cloudwatch), + AuthConfig: aws.AuthConfig(o.Name, o.Cloudwatch.Authentication, op, secrets), + EndpointConfig: endpointConfig(o.Cloudwatch), + RootMixin: common.NewRootMixin("none"), } } @@ -121,6 +151,31 @@ func endpointConfig(cw *obs.Cloudwatch) Element { } } +func groupClassConfig(cw *obs.Cloudwatch) Element { + if cw == nil { + return LogGroupClass{} + } + + //// TODO: avoid regex + //regex := regexp.MustCompile("([a-z0-9])([A-Z])") + //underscored := regex.ReplaceAllString(cw.GroupClass, "${1}_${2}") + //return LogGroupClass{ + // GroupClass: strings.ToLower(underscored), + //} + + // There is only one value that needs to be changed for vector, is it worth it? + //// This is solely for consistency as we are allowing camel-case to be consistent with our older enums + //// I've added the to_upper() to be handled within the vector aws client changes I made + val := cw.GroupClass + if val == InfrequentGroupClassLegacy { + val = InfrequentGroupClass + } + + return LogGroupClass{ + GroupClass: val, + } +} + func NormalizeStreamName(componentID string, inputs []string) Element { vrl := strings.TrimSpace(` .stream_name = "default" diff --git a/internal/generator/vector/output/aws/s3/s3.go b/internal/generator/vector/output/aws/s3/s3.go index 0682fa781..7f55e1ade 100644 --- a/internal/generator/vector/output/aws/s3/s3.go +++ b/internal/generator/vector/output/aws/s3/s3.go @@ -106,6 +106,7 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec return []Element{ template.TemplateRemap(keyPrefixID, inputs, o.S3.KeyPrefix, keyPrefixID, "S3 Key Prefix"), sink, + aws.NewTags(id, o.Cloudwatch), common.NewEncoding(id, common.CodecJSON), common.NewAcknowledgments(id, strategy), common.NewBatch(id, strategy), diff --git a/internal/generator/vector/output/common/aws/tags.go b/internal/generator/vector/output/common/aws/tags.go new file mode 100644 index 000000000..9a34af769 --- /dev/null +++ b/internal/generator/vector/output/common/aws/tags.go @@ -0,0 +1,35 @@ +package aws + +import ( + obs "github.com/openshift/cluster-logging-operator/api/observability/v1" + "github.com/openshift/cluster-logging-operator/internal/generator/framework" +) + +type Tags struct { + Tags map[string]string + ID string +} + +func (t Tags) Name() string { + return "awsTagsTemplate" +} + +func (t Tags) Template() (s string) { + return `{{define "` + t.Name() + `" -}} +[sinks.{{.ID}}.tags] +{{- range $key, $value := .Tags }} +{{ printf "\"%s\" = \"%s\"\n" $key $value }} +{{- end -}} +{{end}}` +} + +// NewTags adds the tags config to CloudWatch or S3 sink +func NewTags(id string, cw *obs.Cloudwatch) framework.Element { + if cw.Tags == nil { + return Tags{} + } + return Tags{ + Tags: cw.Tags, + ID: id, + } +}