Kinesis refactor (#226)

* - fix manifests generator
* - example
* - format fix
* - Crd update

Author: ahma

Makefile

@@ -84,7 +84,7 @@ deploy: manifests
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	cd pkg/sdk && $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=../../config/crd/bases output:webhook:artifacts:config=../../config/webhook
 
 # Run go fmt against code
 fmt:

charts/logging-operator/templates/logging.banzaicloud.io_clusteroutputs.yaml

@@ -0,0 +1,24 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Output
+metadata:
+  name: cloudwatch-output
+  namespace: logging
+spec:
+  cloudwatch:
+    aws_key_id:
+      valueFrom:
+        secretKeyRef:
+          name: logging-s3
+          key: awsAccessKeyId
+    aws_sec_key:
+      valueFrom:
+        secretKeyRef:
+          name: logging-s3
+          key: awsSecretAccesKey
+    stream_name: operator-log-stream
+    region: us-east-1
+    auto_create_stream: true
+    buffer:
+      timekey: 30s
+      timekey_wait: 30s
+      timekey_use_utc: true

charts/logging-operator/templates/logging.banzaicloud.io_outputs.yaml

@@ -6,7 +6,7 @@
  ```
  spec:
    kinesisStream:
-     streamName: example-stream-name
+     stream_name: example-stream-name
      region: us-east-1
      format:
        type: json
@@ -18,7 +18,7 @@
 
 | Variable Name | Type | Required | Default | Description |
 |---|---|---|---|---|
-| kinesisStream | string | Yes | - | Name of the stream to put data.<br> |
+| stream_name | string | Yes | - | Name of the stream to put data.<br> |
 | partition_key | string | No | - | A key to extract partition key from JSON object. Default nil, which means partition key will be generated randomly.<br> |
 | aws_key_id | *secret.Secret | No | - | AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role.<br> |
 | aws_sec_key | *secret.Secret | No | - | AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.<br> |
@@ -30,7 +30,7 @@
 | reset_backoff_if_success | bool | No | - | Boolean, default true. If enabled, when after retrying, the next retrying checks the number of succeeded records on the former batch request and reset exponential backoff if there is any success. Because batch request could be composed by requests across shards, simple exponential backoff for the batch request wouldn't work some cases.<br> |
 | batch_request_max_count | int | No | - | Integer, default 500. The number of max count of making batch request from record chunk. It can't exceed the default value because it's API limit.<br> |
 | batch_request_max_size | int | No | - | Integer. The number of max size of making batch request from record chunk. It can't exceed the default value because it's API limit.<br> |
-| format | *Format | Yes | - | [Format](./format.md)<br> |
+| format | *Format | No | - | [Format](./format.md)<br> |
 | buffer | *Buffer | No | - | [Buffer](./buffer.md)<br> |
 ### Assume Role Credentials
 #### assume_role_credentials

config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml

@@ -21,19 +21,20 @@ import (
 
 // OutputSpec defines the desired state of Output
 type OutputSpec struct {
-	LoggingRef          string                      `json:"loggingRef,omitempty"`
-	S3OutputConfig      *output.S3OutputConfig      `json:"s3,omitempty"`
-	AzureStorage        *output.AzureStorage        `json:"azurestorage,omitempty"`
-	GCSOutput           *output.GCSOutput           `json:"gcs,omitempty"`
-	OSSOutput           *output.OSSOutput           `json:"oss,omitempty"`
-	ElasticsearchOutput *output.ElasticsearchOutput `json:"elasticsearch,omitempty"`
-	LokiOutput          *output.LokiOutput          `json:"loki,omitempty"`
-	SumologicOutput     *output.SumologicOutput     `json:"sumologic,omitempty"`
-	ForwardOutput       *output.ForwardOutput       `json:"forward,omitempty"`
-	FileOutput          *output.FileOutputConfig    `json:"file,omitempty"`
-	NullOutputConfig    *output.NullOutputConfig    `json:"nullout,omitempty"`
-	KafkaOutputConfig   *output.KafkaOutputConfig   `json:"kafka,omitempty"`
-	CloudWatchOutput    *output.CloudWatchOutput    `json:"cloudwatch,omitempty"`
+	LoggingRef                string                            `json:"loggingRef,omitempty"`
+	S3OutputConfig            *output.S3OutputConfig            `json:"s3,omitempty"`
+	AzureStorage              *output.AzureStorage              `json:"azurestorage,omitempty"`
+	GCSOutput                 *output.GCSOutput                 `json:"gcs,omitempty"`
+	OSSOutput                 *output.OSSOutput                 `json:"oss,omitempty"`
+	ElasticsearchOutput       *output.ElasticsearchOutput       `json:"elasticsearch,omitempty"`
+	LokiOutput                *output.LokiOutput                `json:"loki,omitempty"`
+	SumologicOutput           *output.SumologicOutput           `json:"sumologic,omitempty"`
+	ForwardOutput             *output.ForwardOutput             `json:"forward,omitempty"`
+	FileOutput                *output.FileOutputConfig          `json:"file,omitempty"`
+	NullOutputConfig          *output.NullOutputConfig          `json:"nullout,omitempty"`
+	KafkaOutputConfig         *output.KafkaOutputConfig         `json:"kafka,omitempty"`
+	CloudWatchOutput          *output.CloudWatchOutput          `json:"cloudwatch,omitempty"`
+	KinesisStreamOutputConfig *output.KinesisStreamOutputConfig `json:"kinesisStream,omitempty"`
 }
 
 // OutputStatus defines the observed state of Output

config/crd/bases/logging.banzaicloud.io_outputs.yaml

@@ -26,7 +26,7 @@ import (
 // ```
 // spec:
 //   kinesisStream:
-//     streamName: example-stream-name
+//     stream_name: example-stream-name
 //     region: us-east-1
 //     format:
 //       type: json
@@ -39,7 +39,7 @@ type _docKinesisStream interface{}
 type KinesisStreamOutputConfig struct {
 
 	// Name of the stream to put data.
-	KinesisStream string `json:"kinesisStream"`
+	StreamName string `json:"stream_name"`
 
 	// A key to extract partition key from JSON object. Default nil, which means partition key will be generated randomly.
 	PartitionKey string `json:"partition_key,omitempty"`
@@ -75,7 +75,7 @@ type KinesisStreamOutputConfig struct {
 	BatchRequestMaxSize int `json:"batch_request_max_size,omitempty"`
 
 	// +docLink:"Format,./format.md"
-	Format *Format `json:"format"`
+	Format *Format `json:"format,omitempty"`
 	// +docLink:"Buffer,./buffer.md"
 	Buffer *Buffer `json:"buffer,omitempty"`
 }