From a6ec71d69f02f254498d918d756b1ace905a26cc Mon Sep 17 00:00:00 2001
From: David Bremner <dave.bremner2@gmail.com>
Date: Fri, 28 Dec 2018 11:38:24 -0600
Subject: [PATCH 01/49] Update README.md

Remove a duplicate line.
---
 README.md | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/README.md b/README.md
index d684359..a0a1109 100644
--- a/README.md
+++ b/README.md
@@ -57,14 +57,6 @@ Fluent Bit DaemonSet ready to be used with Kafka on a normal Kubernetes Cluster:
 $ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds.yaml
 ```
 
-#### Fluent Bit to Elasticsearch on Minikube
-
-If you are using Minikube for testing purposes, use the following alternative DaemonSet manifest:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds-minikube.yaml
-```
-
 ## Details
 
 The default configuration of Fluent Bit makes sure of the following:

From d326f8e7a0acb3d3696b600bc0763213df6ad901 Mon Sep 17 00:00:00 2001
From: Eduardo Silva <eduardo@treasure-data.com>
Date: Mon, 25 Mar 2019 12:59:12 -0600
Subject: [PATCH 02/49] fix: nginx parser regex

Signed-off-by: Julien Vincent <julienlucvincent@gmail.com>
---
 fluent-bit-config-kafka-rest.yml                 | 2 +-
 output/elasticsearch/fluent-bit-configmap.yaml   | 2 +-
 output/elasticsearch/fluent-bit-ds-minikube.yaml | 2 +-
 output/elasticsearch/fluent-bit-ds.yaml          | 2 +-
 output/kafka/fluent-bit-configmap.yaml           | 2 +-
 output/kafka/fluent-bit-ds-minikube.yaml         | 2 +-
 output/kafka/fluent-bit-ds.yaml                  | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/fluent-bit-config-kafka-rest.yml b/fluent-bit-config-kafka-rest.yml
index cfd3089..606c408 100644
--- a/fluent-bit-config-kafka-rest.yml
+++ b/fluent-bit-config-kafka-rest.yml
@@ -61,7 +61,7 @@ data:
     [PARSER]
         Name   nginx
         Format regex
-        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
         Time_Key time
         Time_Format %d/%b/%Y:%H:%M:%S %z
 
diff --git a/output/elasticsearch/fluent-bit-configmap.yaml b/output/elasticsearch/fluent-bit-configmap.yaml
index 67566cb..ecfde0c 100644
--- a/output/elasticsearch/fluent-bit-configmap.yaml
+++ b/output/elasticsearch/fluent-bit-configmap.yaml
@@ -74,7 +74,7 @@ data:
     [PARSER]
         Name   nginx
         Format regex
-        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
         Time_Key time
         Time_Format %d/%b/%Y:%H:%M:%S %z
 
diff --git a/output/elasticsearch/fluent-bit-ds-minikube.yaml b/output/elasticsearch/fluent-bit-ds-minikube.yaml
index 12a29f2..4729b95 100644
--- a/output/elasticsearch/fluent-bit-ds-minikube.yaml
+++ b/output/elasticsearch/fluent-bit-ds-minikube.yaml
@@ -21,7 +21,7 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit:1.0.4
+        image: fluent/fluent-bit:1.0.5
         imagePullPolicy: Always
         ports:
           - containerPort: 2020
diff --git a/output/elasticsearch/fluent-bit-ds.yaml b/output/elasticsearch/fluent-bit-ds.yaml
index 1427cac..a67304e 100644
--- a/output/elasticsearch/fluent-bit-ds.yaml
+++ b/output/elasticsearch/fluent-bit-ds.yaml
@@ -21,7 +21,7 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit:1.0.4
+        image: fluent/fluent-bit:1.0.5
         imagePullPolicy: Always
         ports:
           - containerPort: 2020
diff --git a/output/kafka/fluent-bit-configmap.yaml b/output/kafka/fluent-bit-configmap.yaml
index 103c7db..ca7feed 100644
--- a/output/kafka/fluent-bit-configmap.yaml
+++ b/output/kafka/fluent-bit-configmap.yaml
@@ -79,7 +79,7 @@ data:
     [PARSER]
         Name   nginx
         Format regex
-        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
         Time_Key time
         Time_Format %d/%b/%Y:%H:%M:%S %z
 
diff --git a/output/kafka/fluent-bit-ds-minikube.yaml b/output/kafka/fluent-bit-ds-minikube.yaml
index e582057..55a9054 100644
--- a/output/kafka/fluent-bit-ds-minikube.yaml
+++ b/output/kafka/fluent-bit-ds-minikube.yaml
@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit:1.0.4
+        image: fluent/fluent-bit:1.0.5
         imagePullPolicy: Always
         ports:
         - containerPort: 2020
diff --git a/output/kafka/fluent-bit-ds.yaml b/output/kafka/fluent-bit-ds.yaml
index 7aea185..70c96a2 100644
--- a/output/kafka/fluent-bit-ds.yaml
+++ b/output/kafka/fluent-bit-ds.yaml
@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit:1.0.4
+        image: fluent/fluent-bit:1.0.5
         imagePullPolicy: Always
         ports:
         - containerPort: 2020

From 531a6a7cfd99fe639efb7251c2ff8b1965f16464 Mon Sep 17 00:00:00 2001
From: Harrison Heck <harrison.heck@linio.com>
Date: Fri, 19 Oct 2018 12:30:18 -0400
Subject: [PATCH 03/49] fix: typo in readme

Signed-off-by: Harrison Heck <harrison.heck@gmail.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d684359..186bb37 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,7 @@ The default configuration of Fluent Bit makes sure of the following:
 
 - Consume all containers logs from the running Node.
 - The [Tail input plugin](http://fluentbit.io/documentation/0.12/input/tail.html) will not append more than __5MB__  into the engine until they are flushed to the Elasticsearch backend. This limit aims to provide a workaround for [backpressure](http://fluentbit.io/documentation/0.13/configuration/backpressure.html) scenarios.
-- The Kubernetes filter will enrigh the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
+- The Kubernetes filter will enrich the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
 - The default backend in the configuration is Elasticsearch set by the [Elasticsearch Output Plugin](http://fluentbit.io/documentation/0.13/output/elasticsearch.html). It uses the Logstash format to ingest the logs. If you need a different Index and Type, please refer to the plugin option and do your own adjustments.
 - There is an option called __Retry_Limit__ set to False that means if Fluent Bit cannot flush the records to Elasticsearch it will re-try indefinitely until it succeeds.
 

From 36b3ab50d358c1b87dc0537fbeba8513684403d9 Mon Sep 17 00:00:00 2001
From: Karl Skewes <karl.skewes@gmail.com>
Date: Sun, 4 Nov 2018 09:02:30 +1300
Subject: [PATCH 04/49] Update labels to Kubernetes recommended naming scheme

Signed-off-by: Karl Skewes <karl.skewes@gmail.com>
---
 fluent-bit-daemonset-kafka-rest.yml              | 8 ++++----
 output/elasticsearch/fluent-bit-configmap.yaml   | 2 +-
 output/elasticsearch/fluent-bit-ds-minikube.yaml | 8 ++++----
 output/elasticsearch/fluent-bit-ds.yaml          | 8 ++++----
 output/kafka/fluent-bit-configmap.yaml           | 2 +-
 output/kafka/fluent-bit-ds-minikube.yaml         | 8 ++++----
 output/kafka/fluent-bit-ds.yaml                  | 8 ++++----
 7 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/fluent-bit-daemonset-kafka-rest.yml b/fluent-bit-daemonset-kafka-rest.yml
index 1ce18d1..92fa14c 100644
--- a/fluent-bit-daemonset-kafka-rest.yml
+++ b/fluent-bit-daemonset-kafka-rest.yml
@@ -4,15 +4,15 @@ metadata:
   name: fluent-bit
   namespace: kube-system
   labels:
-    k8s-app: fluent-bit-logging
-    version: v1
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
     kubernetes.io/cluster-service: "true"
 spec:
   template:
     metadata:
       labels:
-        k8s-app: fluent-bit-logging
-        version: v1
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
         kubernetes.io/cluster-service: "true"
     spec:
       containers:
diff --git a/output/elasticsearch/fluent-bit-configmap.yaml b/output/elasticsearch/fluent-bit-configmap.yaml
index df13d5e..2576a8c 100644
--- a/output/elasticsearch/fluent-bit-configmap.yaml
+++ b/output/elasticsearch/fluent-bit-configmap.yaml
@@ -4,7 +4,7 @@ metadata:
   name: fluent-bit-config
   namespace: logging
   labels:
-    k8s-app: fluent-bit
+    app.kubernetes.io/name: fluent-bit
 data:
   # Configuration files: server, input, filters and output
   # ======================================================
diff --git a/output/elasticsearch/fluent-bit-ds-minikube.yaml b/output/elasticsearch/fluent-bit-ds-minikube.yaml
index f9a2838..dba4082 100644
--- a/output/elasticsearch/fluent-bit-ds-minikube.yaml
+++ b/output/elasticsearch/fluent-bit-ds-minikube.yaml
@@ -4,15 +4,15 @@ metadata:
   name: fluent-bit
   namespace: logging
   labels:
-    k8s-app: fluent-bit-logging
-    version: v1
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
     kubernetes.io/cluster-service: "true"
 spec:
   template:
     metadata:
       labels:
-        k8s-app: fluent-bit-logging
-        version: v1
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
         kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"
diff --git a/output/elasticsearch/fluent-bit-ds.yaml b/output/elasticsearch/fluent-bit-ds.yaml
index 0754df4..9bbe000 100644
--- a/output/elasticsearch/fluent-bit-ds.yaml
+++ b/output/elasticsearch/fluent-bit-ds.yaml
@@ -4,15 +4,15 @@ metadata:
   name: fluent-bit
   namespace: logging
   labels:
-    k8s-app: fluent-bit-logging
-    version: v1
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
     kubernetes.io/cluster-service: "true"
 spec:
   template:
     metadata:
       labels:
-        k8s-app: fluent-bit-logging
-        version: v1
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
         kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"
diff --git a/output/kafka/fluent-bit-configmap.yaml b/output/kafka/fluent-bit-configmap.yaml
index ff5bccb..2085a69 100644
--- a/output/kafka/fluent-bit-configmap.yaml
+++ b/output/kafka/fluent-bit-configmap.yaml
@@ -4,7 +4,7 @@ metadata:
   name: fluent-bit-config
   namespace: logging
   labels:
-    k8s-app: fluent-bit
+    app.kubernetes.io/name: fluent-bit
 data:
   # Configuration files: server, input, filters and output
   # ======================================================
diff --git a/output/kafka/fluent-bit-ds-minikube.yaml b/output/kafka/fluent-bit-ds-minikube.yaml
index 44f71e8..a86e912 100644
--- a/output/kafka/fluent-bit-ds-minikube.yaml
+++ b/output/kafka/fluent-bit-ds-minikube.yaml
@@ -4,8 +4,8 @@ metadata:
   name: fluent-bit
   namespace: logging
   labels:
-    k8s-app: fluent-bit-logging
-    version: v1
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
     kubernetes.io/cluster-service: "true"
 spec:
   updateStrategy:
@@ -13,8 +13,8 @@ spec:
   template:
     metadata:
       labels:
-        k8s-app: fluent-bit-logging
-        version: v1
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
         kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"
diff --git a/output/kafka/fluent-bit-ds.yaml b/output/kafka/fluent-bit-ds.yaml
index 67931b7..f3e1962 100644
--- a/output/kafka/fluent-bit-ds.yaml
+++ b/output/kafka/fluent-bit-ds.yaml
@@ -4,8 +4,8 @@ metadata:
   name: fluent-bit
   namespace: logging
   labels:
-    k8s-app: fluent-bit-logging
-    version: v1
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
     kubernetes.io/cluster-service: "true"
 spec:
   updateStrategy:
@@ -13,8 +13,8 @@ spec:
   template:
     metadata:
       labels:
-        k8s-app: fluent-bit-logging
-        version: v1
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
         kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"

From d3fd620111a6870d29a584cdb89bd20ec376e589 Mon Sep 17 00:00:00 2001
From: "David J. M. Karlsen" <david@davidkarlsen.com>
Date: Wed, 25 Sep 2019 01:50:03 +0200
Subject: [PATCH 05/49] Update README.md

Signed-off-by: David J. M. Karlsen <david@davidkarlsen.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ca2986b..1519cc5 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,7 @@ The default configuration of Fluent Bit makes sure of the following:
 
 - Consume all containers logs from the running Node.
 - The [Tail input plugin](http://fluentbit.io/documentation/0.12/input/tail.html) will not append more than __5MB__  into the engine until they are flushed to the Elasticsearch backend. This limit aims to provide a workaround for [backpressure](http://fluentbit.io/documentation/0.13/configuration/backpressure.html) scenarios.
-- The Kubernetes filter will enrigh the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
+- The Kubernetes filter will enrich the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
 - The default backend in the configuration is Elasticsearch set by the [Elasticsearch Output Plugin](http://fluentbit.io/documentation/0.13/output/elasticsearch.html). It uses the Logstash format to ingest the logs. If you need a different Index and Type, please refer to the plugin option and do your own adjustments.
 - There is an option called __Retry_Limit__ set to False that means if Fluent Bit cannot flush the records to Elasticsearch it will re-try indefinitely until it succeeds.
 

From a8d9bd612cfa261ea533d65715d46a2338301dbc Mon Sep 17 00:00:00 2001
From: Rajesh Rajendran <rjshrjndrn@users.noreply.github.com>
Date: Fri, 10 Jan 2020 07:30:33 +0530
Subject: [PATCH 06/49] Updating api endpoint and selector

It's safe to add apps/v1 to Daemonset as it's stable from 1.9, Which I think almost all are.

Signed-off-by: rjshrjndrn <rjshrjndrn@gmail.com>
---
 output/kafka/fluent-bit-ds.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/output/kafka/fluent-bit-ds.yaml b/output/kafka/fluent-bit-ds.yaml
index 304225a..4884163 100644
--- a/output/kafka/fluent-bit-ds.yaml
+++ b/output/kafka/fluent-bit-ds.yaml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: fluent-bit
@@ -10,6 +10,10 @@ metadata:
 spec:
   updateStrategy:
     type: RollingUpdate
+  selector:
+    matchLabels:
+      k8s-app: fluent-bit-logging
+      version: v1
   template:
     metadata:
       labels:

From 105fa63abb04075b635077fe3b03910bf9c68fc0 Mon Sep 17 00:00:00 2001
From: Igor Yankovsky <iyankovsky.tula@gmail.com>
Date: Tue, 20 Oct 2020 13:49:13 +0300
Subject: [PATCH 07/49] Fix duplication of 'Fluent Bit to Elasticsearch on
 Minikube' section to reference Kafka version

Signed-off-by: Igor Yankovsky <iyankovsky.tula@gmail.com>
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0954f08..3f19926 100644
--- a/README.md
+++ b/README.md
@@ -63,12 +63,12 @@ Fluent Bit DaemonSet ready to be used with Kafka on a normal Kubernetes Cluster:
 $ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds.yaml
 ```
 
-#### Fluent Bit to Elasticsearch on Minikube
+#### Fluent Bit to Kafka on Minikube
 
 If you are using Minikube for testing purposes, use the following alternative DaemonSet manifest:
 
 ```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds-minikube.yaml
+$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds-minikube.yaml
 ```
 
 ## Details

From 3a18cdcafb3ac912fa35ec67946ca05c12f5817d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 08:13:25 +0200
Subject: [PATCH 08/49] Revert "Update README.md"

This reverts commit a6ec71d69f02f254498d918d756b1ace905a26cc.
---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 6a212ea..22c9ad9 100644
--- a/README.md
+++ b/README.md
@@ -65,6 +65,14 @@ Fluent Bit DaemonSet ready to be used with Kafka on a normal Kubernetes Cluster:
 $ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds.yaml
 ```
 
+#### Fluent Bit to Elasticsearch on Minikube
+
+If you are using Minikube for testing purposes, use the following alternative DaemonSet manifest:
+
+```
+$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds-minikube.yaml
+```
+
 ## Details
 
 The default configuration of Fluent Bit makes sure of the following:

From 6f7b856cc5ab66a7809f0357573fe6825885a8a0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 08:30:02 +0200
Subject: [PATCH 09/49] Starts the Kustomize base from the elasticsearch
 example

---
 base/fluent-bit-daemonset.yaml | 63 ++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 base/fluent-bit-daemonset.yaml

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
new file mode 100644
index 0000000..66cfdcc
--- /dev/null
+++ b/base/fluent-bit-daemonset.yaml
@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: logging
+  labels:
+    app.kubernetes.io/name: fluent-bit-logging
+    app.kubernetes.io/version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluent-bit-logging
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: fluent-bit-logging
+        app.kubernetes.io/version: v1
+        kubernetes.io/cluster-service: "true"
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "2020"
+        prometheus.io/path: /api/v1/metrics/prometheus
+    spec:
+      containers:
+      - name: fluent-bit
+        image: fluent/fluent-bit:1.5
+        imagePullPolicy: Always
+        ports:
+          - containerPort: 2020
+        env:
+        - name: FLUENT_ELASTICSEARCH_HOST
+          value: "elasticsearch"
+        - name: FLUENT_ELASTICSEARCH_PORT
+          value: "9200"
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: fluent-bit-config
+          mountPath: /fluent-bit/etc/
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: fluent-bit-config
+        configMap:
+          name: fluent-bit-config
+      serviceAccountName: fluent-bit
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"

From fcd56228b9e9d68311e99f29cda9c9e784dceebe Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 06:51:49 +0200
Subject: [PATCH 10/49] Fixes label inconsistencies

kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
---
 base/fluent-bit-daemonset.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 66cfdcc..67d3d9a 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -4,18 +4,16 @@ metadata:
   name: fluent-bit
   namespace: logging
   labels:
-    app.kubernetes.io/name: fluent-bit-logging
-    app.kubernetes.io/version: v1
+    app.kubernetes.io/name: fluent-bit
     kubernetes.io/cluster-service: "true"
 spec:
   selector:
     matchLabels:
-      k8s-app: fluent-bit-logging
+      app.kubernetes.io/name: fluent-bit
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: fluent-bit-logging
-        app.kubernetes.io/version: v1
+        app.kubernetes.io/name: fluent-bit
         kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"

From 6fa4824acbe0e129cd9c145c79c5b976232f2afb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 06:53:47 +0200
Subject: [PATCH 11/49] Removes deprecated label, kubernetes/kubernetes#72757

---
 base/fluent-bit-daemonset.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 67d3d9a..a922503 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -5,7 +5,6 @@ metadata:
   namespace: logging
   labels:
     app.kubernetes.io/name: fluent-bit
-    kubernetes.io/cluster-service: "true"
 spec:
   selector:
     matchLabels:
@@ -14,7 +13,6 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: fluent-bit
-        kubernetes.io/cluster-service: "true"
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "2020"

From 072a479d00cd979936e1dd785c07b67d3be9a80a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 06:55:36 +0200
Subject: [PATCH 12/49] I think the yaml ordering aimed to keep volumes and
 volumeMounts

near each other
---
 base/fluent-bit-daemonset.yaml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index a922503..78ecbc9 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -18,6 +18,16 @@ spec:
         prometheus.io/port: "2020"
         prometheus.io/path: /api/v1/metrics/prometheus
     spec:
+      serviceAccountName: fluent-bit
+      terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
       containers:
       - name: fluent-bit
         image: fluent/fluent-bit:1.5
@@ -37,7 +47,6 @@ spec:
           readOnly: true
         - name: fluent-bit-config
           mountPath: /fluent-bit/etc/
-      terminationGracePeriodSeconds: 10
       volumes:
       - name: varlog
         hostPath:
@@ -48,12 +57,3 @@ spec:
       - name: fluent-bit-config
         configMap:
           name: fluent-bit-config
-      serviceAccountName: fluent-bit
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      - operator: "Exists"
-        effect: "NoExecute"
-      - operator: "Exists"
-        effect: "NoSchedule"

From e69e738c33fc1484bcdd72dbd420af1a19d4dae9 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 06:56:35 +0200
Subject: [PATCH 13/49] The limit for graceful node shutdown

Would be interesting to keep fluent-bit log forwarding for the duration of shutdown.

https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/
https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown
https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms
---
 base/fluent-bit-daemonset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 78ecbc9..d42e313 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -19,7 +19,7 @@ spec:
         prometheus.io/path: /api/v1/metrics/prometheus
     spec:
       serviceAccountName: fluent-bit
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 25
       tolerations:
       - key: node-role.kubernetes.io/master
         operator: Exists

From 1c2091db3de1bc4df660142a373d67b0482208ba Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 07:14:08 +0200
Subject: [PATCH 14/49] There's no indication that the duplication is needed

https://github.com/fluent/fluent-bit-kubernetes-logging/pull/41#issuecomment-476432473
---
 base/fluent-bit-daemonset.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index d42e313..947af23 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -24,10 +24,8 @@ spec:
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
-      - operator: "Exists"
-        effect: "NoExecute"
-      - operator: "Exists"
-        effect: "NoSchedule"
+      - operator: Exists
+        effect: NoExecute
       containers:
       - name: fluent-bit
         image: fluent/fluent-bit:1.5

From d5186b95dc0914c7538c51578b39fd7c4c34779c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 07:15:15 +0200
Subject: [PATCH 15/49] Removes stuff that is better handled with Kustomize

---
 base/fluent-bit-daemonset.yaml | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 947af23..cf236a6 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: fluent-bit
-  namespace: logging
   labels:
     app.kubernetes.io/name: fluent-bit
 spec:
@@ -28,15 +27,9 @@ spec:
         effect: NoExecute
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit:1.5
-        imagePullPolicy: Always
+        image: fluent/fluent-bit
         ports:
-          - containerPort: 2020
-        env:
-        - name: FLUENT_ELASTICSEARCH_HOST
-          value: "elasticsearch"
-        - name: FLUENT_ELASTICSEARCH_PORT
-          value: "9200"
+        - containerPort: 2020
         volumeMounts:
         - name: varlog
           mountPath: /var/log

From 38b56e88247583b23c46a80b89edb5fa5a5ac543 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 15:18:21 +0200
Subject: [PATCH 16/49] Ports current elasticsearch config to Kustomize

Invdividually overridable files, using "behavior: merge"

Note that with "disableNameSuffixHash: false" (the default)
we can roll back configuration changes.
---
 base/filter-kubernetes.conf    | 11 +++++++
 base/fluent-bit.conf           | 12 ++++++++
 base/input-kubernetes.conf     |  9 ++++++
 base/kustomization.yaml        | 17 +++++++++++
 base/output-elasticsearch.conf |  8 +++++
 base/parsers.conf              | 53 ++++++++++++++++++++++++++++++++++
 6 files changed, 110 insertions(+)
 create mode 100644 base/filter-kubernetes.conf
 create mode 100644 base/fluent-bit.conf
 create mode 100644 base/input-kubernetes.conf
 create mode 100644 base/kustomization.yaml
 create mode 100644 base/output-elasticsearch.conf
 create mode 100644 base/parsers.conf

diff --git a/base/filter-kubernetes.conf b/base/filter-kubernetes.conf
new file mode 100644
index 0000000..2c11809
--- /dev/null
+++ b/base/filter-kubernetes.conf
@@ -0,0 +1,11 @@
+[FILTER]
+    Name                kubernetes
+    Match               kube.*
+    Kube_URL            https://kubernetes.default.svc:443
+    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
+    Kube_Tag_Prefix     kube.var.log.containers.
+    Merge_Log           On
+    Merge_Log_Key       log_processed
+    K8S-Logging.Parser  On
+    K8S-Logging.Exclude Off
diff --git a/base/fluent-bit.conf b/base/fluent-bit.conf
new file mode 100644
index 0000000..0f8e560
--- /dev/null
+++ b/base/fluent-bit.conf
@@ -0,0 +1,12 @@
+[SERVICE]
+    Flush         1
+    Log_Level     info
+    Daemon        off
+    Parsers_File  parsers.conf
+    HTTP_Server   On
+    HTTP_Listen   0.0.0.0
+    HTTP_Port     2020
+
+@INCLUDE input-kubernetes.conf
+@INCLUDE filter-kubernetes.conf
+@INCLUDE output-elasticsearch.conf
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
new file mode 100644
index 0000000..f34b77c
--- /dev/null
+++ b/base/input-kubernetes.conf
@@ -0,0 +1,9 @@
+[INPUT]
+    Name              tail
+    Tag               kube.*
+    Path              /var/log/containers/*.log
+    Parser            docker
+    DB                /var/log/flb_kube.db
+    Mem_Buf_Limit     5MB
+    Skip_Long_Lines   On
+    Refresh_Interval  10
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
new file mode 100644
index 0000000..72b7f11
--- /dev/null
+++ b/base/kustomization.yaml
@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+commonLabels:
+  app.kubernetes.io/name: fluent-bit
+
+resources:
+- fluent-bit-daemonset.yaml
+
+configMapGenerator:
+- name: fluent-bit-config
+  files:
+  - fluent-bit.conf
+  - input-kubernetes.conf
+  - filter-kubernetes.conf
+  - output-elasticsearch.conf
+  - parsers.conf
diff --git a/base/output-elasticsearch.conf b/base/output-elasticsearch.conf
new file mode 100644
index 0000000..bb73378
--- /dev/null
+++ b/base/output-elasticsearch.conf
@@ -0,0 +1,8 @@
+[OUTPUT]
+    Name            es
+    Match           *
+    Host            ${FLUENT_ELASTICSEARCH_HOST}
+    Port            ${FLUENT_ELASTICSEARCH_PORT}
+    Logstash_Format On
+    Replace_Dots    On
+    Retry_Limit     False
diff --git a/base/parsers.conf b/base/parsers.conf
new file mode 100644
index 0000000..eb5059c
--- /dev/null
+++ b/base/parsers.conf
@@ -0,0 +1,53 @@
+[PARSER]
+    Name   apache
+    Format regex
+    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name   apache2
+    Format regex
+    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name   apache_error
+    Format regex
+    Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+[PARSER]
+    Name   nginx
+    Format regex
+    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name   json
+    Format json
+    Time_Key time
+    Time_Format %d/%b/%Y:%H:%M:%S %z
+
+[PARSER]
+    Name        docker
+    Format      json
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L
+    Time_Keep   On
+
+[PARSER]
+    # http://rubular.com/r/tjUt3Awgg4
+    Name cri
+    Format regex
+    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+
+[PARSER]
+    Name        syslog
+    Format      regex
+    Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+    Time_Key    time
+    Time_Format %b %d %H:%M:%S

From e0e0289e5028106d79088028a3695b4ec1515f4d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 15:40:39 +0200
Subject: [PATCH 17/49] Removes redundant suffix and prefix

---
 base/fluent-bit-daemonset.yaml | 6 +++---
 base/kustomization.yaml        | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index cf236a6..04b8283 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -36,7 +36,7 @@ spec:
         - name: varlibdockercontainers
           mountPath: /var/lib/docker/containers
           readOnly: true
-        - name: fluent-bit-config
+        - name: config
           mountPath: /fluent-bit/etc/
       volumes:
       - name: varlog
@@ -45,6 +45,6 @@ spec:
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
-      - name: fluent-bit-config
+      - name: config
         configMap:
-          name: fluent-bit-config
+          name: fluent-bit
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 72b7f11..e5e43b5 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -8,7 +8,7 @@ resources:
 - fluent-bit-daemonset.yaml
 
 configMapGenerator:
-- name: fluent-bit-config
+- name: fluent-bit
   files:
   - fluent-bit.conf
   - input-kubernetes.conf

From 209595111de88358541ac0a222752a2f3a97076c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 15:52:40 +0200
Subject: [PATCH 18/49] Migrates RBAC to Kustomize

---
 base/kustomization.yaml                                    | 3 +++
 fluent-bit-role.yaml => rbac/fluent-bit-role.yaml          | 2 +-
 .../fluent-bit-rolebinding.yaml                            | 4 ++--
 .../fluent-bit-serviceaccount.yaml                         | 1 -
 rbac/kustomization.yaml                                    | 7 +++++++
 5 files changed, 13 insertions(+), 4 deletions(-)
 rename fluent-bit-role.yaml => rbac/fluent-bit-role.yaml (76%)
 rename fluent-bit-role-binding.yaml => rbac/fluent-bit-rolebinding.yaml (73%)
 rename fluent-bit-service-account.yaml => rbac/fluent-bit-serviceaccount.yaml (75%)
 create mode 100644 rbac/kustomization.yaml

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index e5e43b5..32833ce 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -1,10 +1,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: logging
+
 commonLabels:
   app.kubernetes.io/name: fluent-bit
 
 resources:
+- ../rbac
 - fluent-bit-daemonset.yaml
 
 configMapGenerator:
diff --git a/fluent-bit-role.yaml b/rbac/fluent-bit-role.yaml
similarity index 76%
rename from fluent-bit-role.yaml
rename to rbac/fluent-bit-role.yaml
index e55463a..0f72b2e 100644
--- a/fluent-bit-role.yaml
+++ b/rbac/fluent-bit-role.yaml
@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: fluent-bit-read
diff --git a/fluent-bit-role-binding.yaml b/rbac/fluent-bit-rolebinding.yaml
similarity index 73%
rename from fluent-bit-role-binding.yaml
rename to rbac/fluent-bit-rolebinding.yaml
index ff6f258..3c39ff7 100644
--- a/fluent-bit-role-binding.yaml
+++ b/rbac/fluent-bit-rolebinding.yaml
@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: fluent-bit-read
@@ -9,4 +9,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: fluent-bit
-  namespace: logging
+  # set by Kustomize: namespace:
diff --git a/fluent-bit-service-account.yaml b/rbac/fluent-bit-serviceaccount.yaml
similarity index 75%
rename from fluent-bit-service-account.yaml
rename to rbac/fluent-bit-serviceaccount.yaml
index c541d81..886943d 100644
--- a/fluent-bit-service-account.yaml
+++ b/rbac/fluent-bit-serviceaccount.yaml
@@ -2,4 +2,3 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: fluent-bit
-  namespace: logging
diff --git a/rbac/kustomization.yaml b/rbac/kustomization.yaml
new file mode 100644
index 0000000..1ea43cb
--- /dev/null
+++ b/rbac/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- fluent-bit-serviceaccount.yaml
+- fluent-bit-role.yaml
+- fluent-bit-rolebinding.yaml

From 9d649e006d581dc1a3101a0faabb1a6f65b5dfe5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 15:59:39 +0200
Subject: [PATCH 19/49] With image digests we don't depend on a particular
 imagePullPolicy

and prefixed by the tag we document what we pulled
---
 base/fluent-bit-daemonset.yaml | 2 +-
 base/kustomization.yaml        | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 04b8283..19626cb 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -27,7 +27,7 @@ spec:
         effect: NoExecute
       containers:
       - name: fluent-bit
-        image: fluent/fluent-bit
+        image: docker.io/fluent/fluent-bit
         ports:
         - containerPort: 2020
         volumeMounts:
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 32833ce..96d15b1 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -3,6 +3,10 @@ kind: Kustomization
 
 namespace: logging
 
+images:
+- name: docker.io/fluent/fluent-bit
+  newTag: 1.8.3@sha256:10ea2709cef6e7059d980b4969d5f9d753ef97278a817c214cbe9120b1152082
+
 commonLabels:
   app.kubernetes.io/name: fluent-bit
 

From f99ebff28c9179a692ff7a40baafc72b88ac4b3f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 10 Aug 2021 16:37:45 +0200
Subject: [PATCH 20/49] Probes from the helm chart

---
 base/fluent-bit-daemonset.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 19626cb..cb4c0fd 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -29,7 +29,16 @@ spec:
       - name: fluent-bit
         image: docker.io/fluent/fluent-bit
         ports:
-        - containerPort: 2020
+        - name: http
+          containerPort: 2020
+        livenessProbe:
+          httpGet:
+            path: /
+            port: http
+        readinessProbe:
+          httpGet:
+            path: /api/v1/health
+            port: http
         volumeMounts:
         - name: varlog
           mountPath: /var/log

From 3dd95af46d5ff2b67d5b0ab99d7ee737ae6e6d82 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 09:20:50 +0200
Subject: [PATCH 21/49] A decent devloop

Displaying unparsed log entries using
kubectl apply -k base; kubectl rollout status daemonset fluent-bit; kubectl logs -f -l app.kubernetes.io/name=fluent-bit

Issues like #88 and https://github.com/fluent/fluent-bit/issues/1902#issuecomment-579752828
indicate that depending on /var/log/containers symlinks cause
quite a few issues.

/var/log/pods/ is the path stated in
https://github.com/kubernetes/kubernetes/blob/v1.22.0/pkg/kubelet/kuberuntime/kuberuntime_manager.go#L63
and I've verified on GKE cos-containerd, GKE ubuntu-dockerd and k3s
that the path contains the actual files, not symliks.

Also using /var/log/pods makes it trivial to exclude logs from any
container named fluent-bit. Doing so reduces the risk of endless log loops.
---
 base/filter-kubernetes.conf    |  2 +-
 base/fluent-bit-daemonset.yaml | 26 ++++++++++++++++++--------
 base/fluent-bit.conf           |  6 +++---
 base/input-kubernetes.conf     |  9 +++++----
 base/kustomization.yaml        |  2 +-
 base/output-elasticsearch.conf |  8 --------
 base/outputs.conf              |  8 ++++++++
 7 files changed, 36 insertions(+), 25 deletions(-)
 delete mode 100644 base/output-elasticsearch.conf
 create mode 100644 base/outputs.conf

diff --git a/base/filter-kubernetes.conf b/base/filter-kubernetes.conf
index 2c11809..2d237ce 100644
--- a/base/filter-kubernetes.conf
+++ b/base/filter-kubernetes.conf
@@ -4,7 +4,7 @@
     Kube_URL            https://kubernetes.default.svc:443
     Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
-    Kube_Tag_Prefix     kube.var.log.containers.
+    Kube_Tag_Prefix     kube.var.log.pods.
     Merge_Log           On
     Merge_Log_Key       log_processed
     K8S-Logging.Parser  On
diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index cb4c0fd..330c590 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -28,6 +28,9 @@ spec:
       containers:
       - name: fluent-bit
         image: docker.io/fluent/fluent-bit
+        env:
+        - name: LOG_LEVEL
+          value: debug
         ports:
         - name: http
           containerPort: 2020
@@ -40,20 +43,27 @@ spec:
             path: /api/v1/health
             port: http
         volumeMounts:
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
+        - name: varlogpods
+          mountPath: /var/log/pods
           readOnly: true
+        - name: db
+          mountPath: /var/run/fluent-bit
         - name: config
           mountPath: /fluent-bit/etc/
+        # - mountPath: /etc/machine-id
+        #   name: etcmachineid
+        #   readOnly: true
       volumes:
-      - name: varlog
+      - name: varlogpods
         hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
+          path: /var/log/pods
+      - name: db
         hostPath:
-          path: /var/lib/docker/containers
+          path: /var/run/fluent-bit
+      # - name: etcmachineid
+      #   hostPath:
+      #     path: /etc/machine-id
+      #     type: File
       - name: config
         configMap:
           name: fluent-bit
diff --git a/base/fluent-bit.conf b/base/fluent-bit.conf
index 0f8e560..aef5ab5 100644
--- a/base/fluent-bit.conf
+++ b/base/fluent-bit.conf
@@ -1,6 +1,6 @@
 [SERVICE]
     Flush         1
-    Log_Level     info
+    Log_Level     ${LOG_LEVEL}
     Daemon        off
     Parsers_File  parsers.conf
     HTTP_Server   On
@@ -8,5 +8,5 @@
     HTTP_Port     2020
 
 @INCLUDE input-kubernetes.conf
-@INCLUDE filter-kubernetes.conf
-@INCLUDE output-elasticsearch.conf
+#@INCLUDE filter-kubernetes.conf
+@INCLUDE outputs.conf
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index f34b77c..64ca914 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -1,9 +1,10 @@
 [INPUT]
     Name              tail
     Tag               kube.*
-    Path              /var/log/containers/*.log
-    Parser            docker
-    DB                /var/log/flb_kube.db
+    DB                /var/run/fluent-bit/tail-positions.db
     Mem_Buf_Limit     5MB
     Skip_Long_Lines   On
-    Refresh_Interval  10
+    DB.locking        true
+    Path              /var/log/pods/*/*/*.log
+    Exclude_Path      /var/log/pods/*/fluent-bit/*.log
+    #Parser            cri
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 96d15b1..a96cda2 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -20,5 +20,5 @@ configMapGenerator:
   - fluent-bit.conf
   - input-kubernetes.conf
   - filter-kubernetes.conf
-  - output-elasticsearch.conf
+  - outputs.conf
   - parsers.conf
diff --git a/base/output-elasticsearch.conf b/base/output-elasticsearch.conf
deleted file mode 100644
index bb73378..0000000
--- a/base/output-elasticsearch.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-[OUTPUT]
-    Name            es
-    Match           *
-    Host            ${FLUENT_ELASTICSEARCH_HOST}
-    Port            ${FLUENT_ELASTICSEARCH_PORT}
-    Logstash_Format On
-    Replace_Dots    On
-    Retry_Limit     False
diff --git a/base/outputs.conf b/base/outputs.conf
new file mode 100644
index 0000000..e091a68
--- /dev/null
+++ b/base/outputs.conf
@@ -0,0 +1,8 @@
+[OUTPUT]
+    Name             stdout
+    Format           json_lines
+    json_date_key    t
+    json_date_format iso8601
+    # Name    file
+    # Path    /var/run/fluent-bit
+    # File    file-output.txt

From 724c616ab6e7f8e5d69871326329b9303ba13775 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 10:39:00 +0200
Subject: [PATCH 22/49] The docker parser has no effect, but cri has

The .log prop goes away and we get .stream .logtag .message
---
 base/input-kubernetes.conf | 2 +-
 base/parsers.conf          | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index 64ca914..d2d1938 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -7,4 +7,4 @@
     DB.locking        true
     Path              /var/log/pods/*/*/*.log
     Exclude_Path      /var/log/pods/*/fluent-bit/*.log
-    #Parser            cri
+    Parser            cri
diff --git a/base/parsers.conf b/base/parsers.conf
index eb5059c..d7c3ece 100644
--- a/base/parsers.conf
+++ b/base/parsers.conf
@@ -30,13 +30,6 @@
     Time_Key time
     Time_Format %d/%b/%Y:%H:%M:%S %z
 
-[PARSER]
-    Name        docker
-    Format      json
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L
-    Time_Keep   On
-
 [PARSER]
     # http://rubular.com/r/tjUt3Awgg4
     Name cri

From 4b0736e338e7574be5c96639353d9a8bf572c624 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 10:49:55 +0200
Subject: [PATCH 23/49] Allows custom parsers without duplication of the cri
 conf

---
 base/fluent-bit.conf    |  1 +
 base/kustomization.yaml |  3 ++-
 base/outputs.conf       |  3 ---
 base/parser-cri.conf    |  8 +++++++
 base/parsers.conf       | 46 -----------------------------------------
 5 files changed, 11 insertions(+), 50 deletions(-)
 create mode 100644 base/parser-cri.conf

diff --git a/base/fluent-bit.conf b/base/fluent-bit.conf
index aef5ab5..b5ae713 100644
--- a/base/fluent-bit.conf
+++ b/base/fluent-bit.conf
@@ -2,6 +2,7 @@
     Flush         1
     Log_Level     ${LOG_LEVEL}
     Daemon        off
+    Parsers_File  parser-cri.conf
     Parsers_File  parsers.conf
     HTTP_Server   On
     HTTP_Listen   0.0.0.0
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index a96cda2..cf9a0de 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -18,7 +18,8 @@ configMapGenerator:
 - name: fluent-bit
   files:
   - fluent-bit.conf
+  - parser-cri.conf
   - input-kubernetes.conf
   - filter-kubernetes.conf
-  - outputs.conf
   - parsers.conf
+  - outputs.conf
diff --git a/base/outputs.conf b/base/outputs.conf
index e091a68..1cba8d0 100644
--- a/base/outputs.conf
+++ b/base/outputs.conf
@@ -3,6 +3,3 @@
     Format           json_lines
     json_date_key    t
     json_date_format iso8601
-    # Name    file
-    # Path    /var/run/fluent-bit
-    # File    file-output.txt
diff --git a/base/parser-cri.conf b/base/parser-cri.conf
new file mode 100644
index 0000000..2319c15
--- /dev/null
+++ b/base/parser-cri.conf
@@ -0,0 +1,8 @@
+
+[PARSER]
+    # http://rubular.com/r/tjUt3Awgg4
+    Name cri
+    Format regex
+    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
diff --git a/base/parsers.conf b/base/parsers.conf
index d7c3ece..e69de29 100644
--- a/base/parsers.conf
+++ b/base/parsers.conf
@@ -1,46 +0,0 @@
-[PARSER]
-    Name   apache
-    Format regex
-    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-    Time_Key time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name   apache2
-    Format regex
-    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-    Time_Key time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name   apache_error
-    Format regex
-    Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
-
-[PARSER]
-    Name   nginx
-    Format regex
-    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
-    Time_Key time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    Name   json
-    Format json
-    Time_Key time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-[PARSER]
-    # http://rubular.com/r/tjUt3Awgg4
-    Name cri
-    Format regex
-    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
-
-[PARSER]
-    Name        syslog
-    Format      regex
-    Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
-    Time_Key    time
-    Time_Format %b %d %H:%M:%S

From 1786335db0069effe00680325a409f4fc9f20c03 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 10:51:12 +0200
Subject: [PATCH 24/49] Uses the same key for log content as with unparsed logs

Based on https://github.com/microsoft/fluentbit-containerd-cri-o-json-log
---
 base/parser-cri.conf | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/base/parser-cri.conf b/base/parser-cri.conf
index 2319c15..2fad4d5 100644
--- a/base/parser-cri.conf
+++ b/base/parser-cri.conf
@@ -1,8 +1,7 @@
 
 [PARSER]
-    # http://rubular.com/r/tjUt3Awgg4
-    Name cri
-    Format regex
-    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+    Name        cri
+    Format      regex
+    Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
     Time_Key    time
     Time_Format %Y-%m-%dT%H:%M:%S.%L%z

From 9b714d4d29df96b8f66cd7b1ecea41ddc68582f4 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 11:09:02 +0200
Subject: [PATCH 25/49] Seems like something you need to tweak

---
 base/fluent-bit-daemonset.yaml | 2 ++
 base/input-kubernetes.conf     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluent-bit-daemonset.yaml
index 330c590..e3c3ef7 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluent-bit-daemonset.yaml
@@ -31,6 +31,8 @@ spec:
         env:
         - name: LOG_LEVEL
           value: debug
+        - name: MEM_BUF_LIMIT
+          value: 5MB
         ports:
         - name: http
           containerPort: 2020
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index d2d1938..483d4e1 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -2,7 +2,7 @@
     Name              tail
     Tag               kube.*
     DB                /var/run/fluent-bit/tail-positions.db
-    Mem_Buf_Limit     5MB
+    Mem_Buf_Limit     ${MEM_BUF_LIMIT}
     Skip_Long_Lines   On
     DB.locking        true
     Path              /var/log/pods/*/*/*.log

From cbb9daec3fbe0148b0e388091bafa9a1335cf1cd Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 13:34:18 +0200
Subject: [PATCH 26/49] Adapts tag regex to path /var/log/pods

I find no benefits with the stricter regexes used in most examples,
as long as paths are well defined.
---
 base/input-kubernetes.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index 483d4e1..026f1f5 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -1,6 +1,5 @@
 [INPUT]
     Name              tail
-    Tag               kube.*
     DB                /var/run/fluent-bit/tail-positions.db
     Mem_Buf_Limit     ${MEM_BUF_LIMIT}
     Skip_Long_Lines   On
@@ -8,3 +7,5 @@
     Path              /var/log/pods/*/*/*.log
     Exclude_Path      /var/log/pods/*/fluent-bit/*.log
     Parser            cri
+    Tag               kube.<namespace_name>.<pod_name>.<pod_uid>.<container_name>
+    Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_uid>[^_/]+)/(?<container_name>[^/]+)/.*

From 70562ddbe0f0a871593028bafd6c7909986b7454 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 13:36:17 +0200
Subject: [PATCH 27/49] Msgpack format is more useful for devloops because it
 includes tags

Looks like the default format of the file output plugin
---
 base/outputs.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/outputs.conf b/base/outputs.conf
index 1cba8d0..fd7c1e3 100644
--- a/base/outputs.conf
+++ b/base/outputs.conf
@@ -1,5 +1,5 @@
 [OUTPUT]
     Name             stdout
-    Format           json_lines
+    Format           msgpack
     json_date_key    t
     json_date_format iso8601

From a5c7b19d94c46b0dddf4965d313c70dbcfab5722 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 15:46:43 +0200
Subject: [PATCH 28/49] Loki with ephemeral storage

Config from
https://grafana.com/docs/loki/next/configuration/examples/#complete-local-config
---
 loki/kustomization.yaml         | 23 +++++++++++++
 loki/loki-headless-service.yaml | 12 +++++++
 loki/loki-service.yaml          | 15 +++++++++
 loki/loki-statefulset.yaml      | 60 +++++++++++++++++++++++++++++++++
 loki/loki.yaml                  | 37 ++++++++++++++++++++
 loki/outputs-loki.conf          |  8 +++++
 6 files changed, 155 insertions(+)
 create mode 100644 loki/kustomization.yaml
 create mode 100644 loki/loki-headless-service.yaml
 create mode 100644 loki/loki-service.yaml
 create mode 100644 loki/loki-statefulset.yaml
 create mode 100644 loki/loki.yaml
 create mode 100644 loki/outputs-loki.conf

diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
new file mode 100644
index 0000000..f150a4f
--- /dev/null
+++ b/loki/kustomization.yaml
@@ -0,0 +1,23 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+images:
+- name: docker.io/grafana/loki
+  newTag: 2.3.0@sha256:bbf6dbf3264af939a541b6f3c014cba21a2bdc8f22cb7962eee7e9048b41ea5e
+
+resources:
+- ../base
+- loki-service.yaml
+- loki-headless-service.yaml
+- loki-statefulset.yaml
+
+secretGenerator:
+- name: loki
+  files:
+  - loki.yaml
+
+configMapGenerator:
+- name: fluent-bit
+  behavior: merge
+  files:
+  - outputs.conf=outputs-loki.conf
diff --git a/loki/loki-headless-service.yaml b/loki/loki-headless-service.yaml
new file mode 100644
index 0000000..7dadef5
--- /dev/null
+++ b/loki/loki-headless-service.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki-headless
+spec:
+  clusterIP: None
+  ports:
+  - name: http-metrics
+    port: 3100
+    targetPort: http-metrics
+  selector:
+    app: loki
diff --git a/loki/loki-service.yaml b/loki/loki-service.yaml
new file mode 100644
index 0000000..0411b00
--- /dev/null
+++ b/loki/loki-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  labels:
+    app: loki
+spec:
+  type: ClusterIP
+  ports:
+  - name: http-metrics
+    port: 3100
+    protocol: TCP
+    targetPort: http-metrics
+  selector:
+    app: loki
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
new file mode 100644
index 0000000..d1c64eb
--- /dev/null
+++ b/loki/loki-statefulset.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: loki
+  labels:
+    app: loki
+spec:
+  replicas: 1
+  podManagementPolicy: OrderedReady
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: loki
+  serviceName: loki-headless
+  template:
+    metadata:
+      labels:
+        app: loki
+      annotations:
+        prometheus.io/port: http-metrics
+        prometheus.io/scrape: "true"
+    spec:
+      #serviceAccountName: loki
+      securityContext:
+        fsGroup: 10001
+        runAsGroup: 10001
+        runAsNonRoot: true
+        runAsUser: 10001
+      terminationGracePeriodSeconds: 25
+      containers:
+      - name: loki
+        image: grafana/loki:2.3.0
+        securityContext:
+          readOnlyRootFilesystem: true
+        args:
+        - -config.file=/etc/loki/loki.yaml
+        ports:
+        - name: http-metrics
+          containerPort: 3100
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /ready
+            port: http-metrics
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: http-metrics
+        volumeMounts:
+        - name: config
+          mountPath: /etc/loki
+        - name: storage
+          mountPath: /data
+      volumes:
+      - name: config
+        secret:
+          secretName: loki
+      - name: storage
+        emptyDir: {}
diff --git a/loki/loki.yaml b/loki/loki.yaml
new file mode 100644
index 0000000..67c7c98
--- /dev/null
+++ b/loki/loki.yaml
@@ -0,0 +1,37 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+schema_config:
+  configs:
+  - from: 2021-08-11
+    store: boltdb
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /data/index
+
+  filesystem:
+    directory: /data/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
diff --git a/loki/outputs-loki.conf b/loki/outputs-loki.conf
new file mode 100644
index 0000000..9602877
--- /dev/null
+++ b/loki/outputs-loki.conf
@@ -0,0 +1,8 @@
+[OUTPUT]
+    Name                   loki
+    Match                  *
+    Host                   loki
+    Port                   3100
+    Labels                 job=fluentbit
+    Label_Keys             $namespace, $pod_name, $pod_uid, $container_name
+    Auto_Kubernetes_Labels on

From d99ccdbc3f60ebf85c3961b00865fdce971f942b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 17:01:09 +0200
Subject: [PATCH 29/49] Easier to follow with the constant length UUID first

---
 base/input-kubernetes.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index 026f1f5..5151b67 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -7,5 +7,5 @@
     Path              /var/log/pods/*/*/*.log
     Exclude_Path      /var/log/pods/*/fluent-bit/*.log
     Parser            cri
-    Tag               kube.<namespace_name>.<pod_name>.<pod_uid>.<container_name>
+    Tag               kube.<pod_uid>.<namespace_name>.<pod_name>.<container_name>
     Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_uid>[^_/]+)/(?<container_name>[^/]+)/.*

From 0da56f8396a48b99fa941e9d24cbf63b719ed270 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 11 Aug 2021 17:29:10 +0200
Subject: [PATCH 30/49] Now includes container metadata and pod labels

---
 base/filter-kubernetes.conf | 16 ++++++++--------
 base/fluent-bit.conf        |  2 +-
 base/input-kubernetes.conf  |  4 ++--
 base/outputs.conf           |  3 ++-
 base/parser-cri.conf        |  5 +++++
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/base/filter-kubernetes.conf b/base/filter-kubernetes.conf
index 2d237ce..33d62be 100644
--- a/base/filter-kubernetes.conf
+++ b/base/filter-kubernetes.conf
@@ -1,11 +1,11 @@
+
 [FILTER]
     Name                kubernetes
     Match               kube.*
-    Kube_URL            https://kubernetes.default.svc:443
-    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
-    Kube_Tag_Prefix     kube.var.log.pods.
-    Merge_Log           On
-    Merge_Log_Key       log_processed
-    K8S-Logging.Parser  On
-    K8S-Logging.Exclude Off
+    Kube_Tag_Prefix     kube.
+    Regex_Parser        kube-tag
+    # Merge_Log           On
+    # Merge_Log_Key       log_processed
+    # K8S-Logging.Parser  On
+    # K8S-Logging.Exclude Off
+    Annotations         Off
diff --git a/base/fluent-bit.conf b/base/fluent-bit.conf
index b5ae713..2c536d6 100644
--- a/base/fluent-bit.conf
+++ b/base/fluent-bit.conf
@@ -9,5 +9,5 @@
     HTTP_Port     2020
 
 @INCLUDE input-kubernetes.conf
-#@INCLUDE filter-kubernetes.conf
+@INCLUDE filter-kubernetes.conf
 @INCLUDE outputs.conf
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index 5151b67..d0e4437 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -7,5 +7,5 @@
     Path              /var/log/pods/*/*/*.log
     Exclude_Path      /var/log/pods/*/fluent-bit/*.log
     Parser            cri
-    Tag               kube.<pod_uid>.<namespace_name>.<pod_name>.<container_name>
-    Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_uid>[^_/]+)/(?<container_name>[^/]+)/.*
+    Tag               kube.<pod_id>.<namespace_name>.<pod_name>.<container_name>
+    Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_id>[^_/]+)/(?<container_name>[^/]+)/.*
diff --git a/base/outputs.conf b/base/outputs.conf
index fd7c1e3..fa4ac9e 100644
--- a/base/outputs.conf
+++ b/base/outputs.conf
@@ -1,5 +1,6 @@
 [OUTPUT]
     Name             stdout
-    Format           msgpack
+    # Switch to format msgpack to see the tags
+    Format           json_lines
     json_date_key    t
     json_date_format iso8601
diff --git a/base/parser-cri.conf b/base/parser-cri.conf
index 2fad4d5..635c71a 100644
--- a/base/parser-cri.conf
+++ b/base/parser-cri.conf
@@ -5,3 +5,8 @@
     Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
     Time_Key    time
     Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+
+[PARSER]
+    Name        kube-tag
+    Format      regex
+    Regex       ^(?<pod_id>[^.]+)\.(?<namespace_name>[^.]+)\.(?<pod_name>[^.]+)\.(?<container_name>[^.]+)$

From efb12aca1385404c2fe7d4ab47d08b025e0a92e2 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 12 Aug 2021 06:14:12 +0200
Subject: [PATCH 31/49] Current from the helm chart

https://github.com/grafana/helm-charts/blob/main/charts/loki/values.yaml
---
 loki/loki.yaml | 49 +++++++++++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/loki/loki.yaml b/loki/loki.yaml
index 67c7c98..ab3ed85 100644
--- a/loki/loki.yaml
+++ b/loki/loki.yaml
@@ -1,37 +1,50 @@
 auth_enabled: false
 
-server:
-  http_listen_port: 3100
-
 ingester:
+  chunk_idle_period: 3m
+  chunk_block_size: 262144
+  chunk_retain_period: 1m
+  max_transfer_retries: 0
   lifecycler:
-    address: 127.0.0.1
     ring:
       kvstore:
         store: inmemory
       replication_factor: 1
-    final_sleep: 0s
-  chunk_idle_period: 5m
-  chunk_retain_period: 30s
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
 
 schema_config:
   configs:
-  - from: 2021-08-11
-    store: boltdb
+  - from: 2020-10-24
+    store: boltdb-shipper
     object_store: filesystem
     schema: v11
     index:
       prefix: index_
-      period: 168h
+      period: 24h
 
-storage_config:
-  boltdb:
-    directory: /data/index
+server:
+  http_listen_port: 3100
 
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /data/loki/boltdb-shipper-active
+    cache_location: /data/loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
   filesystem:
-    directory: /data/chunks
+    directory: /data/loki/chunks
 
-limits_config:
-  enforce_metric_name: false
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+compactor:
+  working_directory: /data/loki/boltdb-shipper-compactor
+  shared_store: filesystem

From 6d3e74a184f9cc469fca6fb1060e048774397b65 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 13 Aug 2021 11:52:48 +0200
Subject: [PATCH 32/49] Ephemeral loki setup

---
 loki/kustomization.yaml         |  6 ++-
 loki/loki-headless-service.yaml |  9 ++--
 loki/loki-service.yaml          |  8 +++-
 loki/loki-statefulset.yaml      | 77 ++++++++++++++++++++++++++-------
 loki/loki.yaml                  | 50 ---------------------
 loki/standalone.config.yaml     | 68 +++++++++++++++++++++++++++++
 6 files changed, 147 insertions(+), 71 deletions(-)
 delete mode 100644 loki/loki.yaml
 create mode 100644 loki/standalone.config.yaml

diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index f150a4f..64df08a 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -14,10 +14,14 @@ resources:
 secretGenerator:
 - name: loki
   files:
-  - loki.yaml
+  - standalone.config.yaml
+  #- distributed-s3-minio.config.yaml
+  options:
+    disableNameSuffixHash: true
 
 configMapGenerator:
 - name: fluent-bit
   behavior: merge
   files:
   - outputs.conf=outputs-loki.conf
+  - labelmap.json
diff --git a/loki/loki-headless-service.yaml b/loki/loki-headless-service.yaml
index 7dadef5..db87f92 100644
--- a/loki/loki-headless-service.yaml
+++ b/loki/loki-headless-service.yaml
@@ -2,11 +2,14 @@ apiVersion: v1
 kind: Service
 metadata:
   name: loki-headless
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
+  publishNotReadyAddresses: true
   clusterIP: None
   ports:
-  - name: http-metrics
-    port: 3100
-    targetPort: http-metrics
+  - name: gossip
+    port: 7946
+    targetPort: gossip
   selector:
     app: loki
diff --git a/loki/loki-service.yaml b/loki/loki-service.yaml
index 0411b00..d71af69 100644
--- a/loki/loki-service.yaml
+++ b/loki/loki-service.yaml
@@ -7,9 +7,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-  - name: http-metrics
+  - name: http
     port: 3100
     protocol: TCP
-    targetPort: http-metrics
+    targetPort: http
+  - name: grpc
+    port: 9096
+    protocol: TCP
+    targetPort: grpc
   selector:
     app: loki
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index d1c64eb..6586975 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -5,10 +5,10 @@ metadata:
   labels:
     app: loki
 spec:
-  replicas: 1
-  podManagementPolicy: OrderedReady
+  replicas: 1 # should match the choice of CONFIG below
+  podManagementPolicy: Parallel
   updateStrategy:
-    type: RollingUpdate
+   type: RollingUpdate
   selector:
     matchLabels:
       app: loki
@@ -18,7 +18,7 @@ spec:
       labels:
         app: loki
       annotations:
-        prometheus.io/port: http-metrics
+        prometheus.io/port: http
         prometheus.io/scrape: "true"
     spec:
       #serviceAccountName: loki
@@ -27,34 +27,81 @@ spec:
         runAsGroup: 10001
         runAsNonRoot: true
         runAsUser: 10001
-      terminationGracePeriodSeconds: 25
+      #terminationGracePeriodSeconds: 25
+      #containers:
+      ### for development
+      terminationGracePeriodSeconds: 1
+      shareProcessNamespace: true
       containers:
+      - name: debug
+        image: curlimages/curl:7.78.0@sha256:a37d88a5b626208146c6d58f75b74a93c35dfc00b6b4676c4e9d2c166ea41954
+        command:
+        - /bin/sh
+        - -ce
+        - |
+          while true; do
+            sleep 10
+            getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
+            curl -s http://localhost:3100/ready || true
+            curl -s http://localhost:3100/metrics | grep 'log_messages_total{' || true
+          done
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      ### for development end
       - name: loki
         image: grafana/loki:2.3.0
         securityContext:
           readOnlyRootFilesystem: true
         args:
-        - -config.file=/etc/loki/loki.yaml
+        - -config.file=/etc/loki/$(CONFIG).config.yaml
+        - -config.expand-env=true
+        - -print-config-stderr=true
+        #- -log.level=debug
+        - -memberlist.transport-debug=true
+        env:
+        - name: CONFIG
+          value: standalone
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         ports:
-        - name: http-metrics
+        - name: http
           containerPort: 3100
           protocol: TCP
-        livenessProbe:
-          httpGet:
-            path: /ready
-            port: http-metrics
+        - name: grpc
+          containerPort: 9096
+          protocol: TCP
+        - name: gossip
+          containerPort: 7946
+          protocol: TCP
+        # livenessProbe:
+        #   httpGet:
+        #     path: /ready
+        #     port: http
         readinessProbe:
           httpGet:
             path: /ready
-            port: http-metrics
+            port: http
         volumeMounts:
         - name: config
           mountPath: /etc/loki
-        - name: storage
-          mountPath: /data
+        - name: data
+          mountPath: /loki
       volumes:
       - name: config
         secret:
           secretName: loki
-      - name: storage
+      - name: data
         emptyDir: {}
diff --git a/loki/loki.yaml b/loki/loki.yaml
deleted file mode 100644
index ab3ed85..0000000
--- a/loki/loki.yaml
+++ /dev/null
@@ -1,50 +0,0 @@
-auth_enabled: false
-
-ingester:
-  chunk_idle_period: 3m
-  chunk_block_size: 262144
-  chunk_retain_period: 1m
-  max_transfer_retries: 0
-  lifecycler:
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-
-limits_config:
-  enforce_metric_name: false
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-
-schema_config:
-  configs:
-  - from: 2020-10-24
-    store: boltdb-shipper
-    object_store: filesystem
-    schema: v11
-    index:
-      prefix: index_
-      period: 24h
-
-server:
-  http_listen_port: 3100
-
-storage_config:
-  boltdb_shipper:
-    active_index_directory: /data/loki/boltdb-shipper-active
-    cache_location: /data/loki/boltdb-shipper-cache
-    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: filesystem
-  filesystem:
-    directory: /data/loki/chunks
-
-chunk_store_config:
-  max_look_back_period: 0s
-
-table_manager:
-  retention_deletes_enabled: false
-  retention_period: 0s
-
-compactor:
-  working_directory: /data/loki/boltdb-shipper-compactor
-  shared_store: filesystem
diff --git a/loki/standalone.config.yaml b/loki/standalone.config.yaml
new file mode 100644
index 0000000..cebf505
--- /dev/null
+++ b/loki/standalone.config.yaml
@@ -0,0 +1,68 @@
+# For more suggested overrides see https://github.com/grafana/loki/blob/v2.3.0/cmd/loki/loki-local-config.yaml
+# and of course https://github.com/grafana/loki/blob/v2.3.0/docs/sources/configuration/_index.md
+
+target: ingester,compactor,querier,query-frontend,table-manager
+
+auth_enabled: false
+
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  lifecycler:
+    address: 0.0.0.0
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  wal:
+    enabled: true
+    dir: /loki/wal
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+  - from: 2021-08-13
+    store: boltdb-shipper
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 1h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+# ruler:
+#   storage:
+#     type: local
+#     local:
+#       directory: /loki/rules
+#   rule_path: /loki/rules-temp
+#   alertmanager_url: http://localhost:9093
+#   ring:
+#     kvstore:
+#       store: inmemory
+#   enable_api: true

From 0ace3c10b179e66405148ebba5a834818fde6756 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 13 Aug 2021 14:35:31 +0200
Subject: [PATCH 33/49] Dropping the dash because it's used as separator
 everywhere else

---
 ...aemonset.yaml => fluentbit-daemonset.yaml} | 24 +++++++++++--------
 base/{fluent-bit.conf => fluentbit.conf}      |  0
 base/input-kubernetes.conf                    |  4 ++--
 base/kustomization.yaml                       |  8 +++----
 loki/kustomization.yaml                       |  2 +-
 loki/loki-statefulset.yaml                    |  3 ++-
 ...uent-bit-role.yaml => fluentbit-role.yaml} |  2 +-
 ...inding.yaml => fluentbit-rolebinding.yaml} |  6 ++---
 ...unt.yaml => fluentbit-serviceaccount.yaml} |  2 +-
 rbac/kustomization.yaml                       |  6 ++---
 10 files changed, 31 insertions(+), 26 deletions(-)
 rename base/{fluent-bit-daemonset.yaml => fluentbit-daemonset.yaml} (76%)
 rename base/{fluent-bit.conf => fluentbit.conf} (100%)
 rename rbac/{fluent-bit-role.yaml => fluentbit-role.yaml} (87%)
 rename rbac/{fluent-bit-rolebinding.yaml => fluentbit-rolebinding.yaml} (75%)
 rename rbac/{fluent-bit-serviceaccount.yaml => fluentbit-serviceaccount.yaml} (70%)

diff --git a/base/fluent-bit-daemonset.yaml b/base/fluentbit-daemonset.yaml
similarity index 76%
rename from base/fluent-bit-daemonset.yaml
rename to base/fluentbit-daemonset.yaml
index e3c3ef7..6add83f 100644
--- a/base/fluent-bit-daemonset.yaml
+++ b/base/fluentbit-daemonset.yaml
@@ -1,23 +1,23 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluent-bit
+  name: fluentbit
   labels:
-    app.kubernetes.io/name: fluent-bit
+    app.kubernetes.io/name: fluentbit
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: fluent-bit
+      app.kubernetes.io/name: fluentbit
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: fluent-bit
+        app.kubernetes.io/name: fluentbit
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "2020"
         prometheus.io/path: /api/v1/metrics/prometheus
     spec:
-      serviceAccountName: fluent-bit
+      serviceAccountName: fluentbit
       terminationGracePeriodSeconds: 25
       tolerations:
       - key: node-role.kubernetes.io/master
@@ -26,8 +26,12 @@ spec:
       - operator: Exists
         effect: NoExecute
       containers:
-      - name: fluent-bit
+      - name: fluentbit
         image: docker.io/fluent/fluent-bit
+        command:
+        - /fluent-bit/bin/fluent-bit
+        - -c
+        - /etc/fluentbit/fluentbit.conf
         env:
         - name: LOG_LEVEL
           value: debug
@@ -49,9 +53,9 @@ spec:
           mountPath: /var/log/pods
           readOnly: true
         - name: db
-          mountPath: /var/run/fluent-bit
+          mountPath: /var/run/fluentbit
         - name: config
-          mountPath: /fluent-bit/etc/
+          mountPath: /etc/fluentbit
         # - mountPath: /etc/machine-id
         #   name: etcmachineid
         #   readOnly: true
@@ -61,11 +65,11 @@ spec:
           path: /var/log/pods
       - name: db
         hostPath:
-          path: /var/run/fluent-bit
+          path: /var/run/fluentbit
       # - name: etcmachineid
       #   hostPath:
       #     path: /etc/machine-id
       #     type: File
       - name: config
         configMap:
-          name: fluent-bit
+          name: fluentbit
diff --git a/base/fluent-bit.conf b/base/fluentbit.conf
similarity index 100%
rename from base/fluent-bit.conf
rename to base/fluentbit.conf
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index d0e4437..d457ae3 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -1,11 +1,11 @@
 [INPUT]
     Name              tail
-    DB                /var/run/fluent-bit/tail-positions.db
+    DB                /var/run/fluentbit/tail-positions.db
     Mem_Buf_Limit     ${MEM_BUF_LIMIT}
     Skip_Long_Lines   On
     DB.locking        true
     Path              /var/log/pods/*/*/*.log
-    Exclude_Path      /var/log/pods/*/fluent-bit/*.log
+    Exclude_Path      /var/log/pods/*/fluentbit/*.log
     Parser            cri
     Tag               kube.<pod_id>.<namespace_name>.<pod_name>.<container_name>
     Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_id>[^_/]+)/(?<container_name>[^/]+)/.*
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index cf9a0de..ed6e5cd 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -8,16 +8,16 @@ images:
   newTag: 1.8.3@sha256:10ea2709cef6e7059d980b4969d5f9d753ef97278a817c214cbe9120b1152082
 
 commonLabels:
-  app.kubernetes.io/name: fluent-bit
+  app.kubernetes.io/name: fluentbit
 
 resources:
 - ../rbac
-- fluent-bit-daemonset.yaml
+- fluentbit-daemonset.yaml
 
 configMapGenerator:
-- name: fluent-bit
+- name: fluentbit
   files:
-  - fluent-bit.conf
+  - fluentbit.conf
   - parser-cri.conf
   - input-kubernetes.conf
   - filter-kubernetes.conf
diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index 64df08a..8265a83 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -20,7 +20,7 @@ secretGenerator:
     disableNameSuffixHash: true
 
 configMapGenerator:
-- name: fluent-bit
+- name: fluentbit
   behavior: merge
   files:
   - outputs.conf=outputs-loki.conf
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index 6586975..4106326 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -39,8 +39,9 @@ spec:
         - /bin/sh
         - -ce
         - |
+          echo "Restart Loki using: kubectl -n $POD_NAMESPACE exec -c loki $POD_NAME -- pkill /usr/bin/loki"
           while true; do
-            sleep 10
+            sleep 30
             getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
             curl -s http://localhost:3100/ready || true
             curl -s http://localhost:3100/metrics | grep 'log_messages_total{' || true
diff --git a/rbac/fluent-bit-role.yaml b/rbac/fluentbit-role.yaml
similarity index 87%
rename from rbac/fluent-bit-role.yaml
rename to rbac/fluentbit-role.yaml
index 0f72b2e..1332b96 100644
--- a/rbac/fluent-bit-role.yaml
+++ b/rbac/fluentbit-role.yaml
@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: fluent-bit-read
+  name: fluentbit-read
 rules:
 - apiGroups: [""]
   resources:
diff --git a/rbac/fluent-bit-rolebinding.yaml b/rbac/fluentbit-rolebinding.yaml
similarity index 75%
rename from rbac/fluent-bit-rolebinding.yaml
rename to rbac/fluentbit-rolebinding.yaml
index 3c39ff7..3a25a7f 100644
--- a/rbac/fluent-bit-rolebinding.yaml
+++ b/rbac/fluentbit-rolebinding.yaml
@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: fluent-bit-read
+  name: fluentbit-read
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: fluent-bit-read
+  name: fluentbit-read
 subjects:
 - kind: ServiceAccount
-  name: fluent-bit
+  name: fluentbit
   # set by Kustomize: namespace:
diff --git a/rbac/fluent-bit-serviceaccount.yaml b/rbac/fluentbit-serviceaccount.yaml
similarity index 70%
rename from rbac/fluent-bit-serviceaccount.yaml
rename to rbac/fluentbit-serviceaccount.yaml
index 886943d..1036a0a 100644
--- a/rbac/fluent-bit-serviceaccount.yaml
+++ b/rbac/fluentbit-serviceaccount.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: fluent-bit
+  name: fluentbit
diff --git a/rbac/kustomization.yaml b/rbac/kustomization.yaml
index 1ea43cb..2ab92f7 100644
--- a/rbac/kustomization.yaml
+++ b/rbac/kustomization.yaml
@@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- fluent-bit-serviceaccount.yaml
-- fluent-bit-role.yaml
-- fluent-bit-rolebinding.yaml
+- fluentbit-serviceaccount.yaml
+- fluentbit-role.yaml
+- fluentbit-rolebinding.yaml

From 24fb8c43ea745f78def9703e94f0e2980e64c5ae Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 13 Aug 2021 18:37:58 +0200
Subject: [PATCH 34/49] A dev stack with loki, minimal parsing upfront

---
 base/input-kubernetes.conf                    |  3 ++-
 loki/kustomization.yaml                       |  3 ++-
 loki/loki-headless-service.yaml               |  7 +++++-
 loki/loki-ingest-service.yaml                 | 15 +++++++++++++
 ...i-service.yaml => loki-query-service.yaml} |  6 ++---
 loki/loki-statefulset.yaml                    | 22 ++++++++++++++-----
 loki/outputs-loki.conf                        | 17 ++++++++------
 loki/standalone.config.yaml                   |  8 ++++++-
 8 files changed, 61 insertions(+), 20 deletions(-)
 create mode 100644 loki/loki-ingest-service.yaml
 rename loki/{loki-service.yaml => loki-query-service.yaml} (73%)

diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index d457ae3..ca486a5 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -5,7 +5,8 @@
     Skip_Long_Lines   On
     DB.locking        true
     Path              /var/log/pods/*/*/*.log
-    Exclude_Path      /var/log/pods/*/fluentbit/*.log
+    # We could exclude loki using K8S-Logging.Exclude bit for fluentbit we want to avoid loops regardless of filters
+    Exclude_Path      /var/log/pods/*/fluentbit/*.log,/var/log/pods/*/loki/*.log
     Parser            cri
     Tag               kube.<pod_id>.<namespace_name>.<pod_name>.<container_name>
     Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_id>[^_/]+)/(?<container_name>[^/]+)/.*
diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index 8265a83..4ceafc2 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -7,7 +7,8 @@ images:
 
 resources:
 - ../base
-- loki-service.yaml
+- loki-ingest-service.yaml
+- loki-query-service.yaml
 - loki-headless-service.yaml
 - loki-statefulset.yaml
 
diff --git a/loki/loki-headless-service.yaml b/loki/loki-headless-service.yaml
index db87f92..f228804 100644
--- a/loki/loki-headless-service.yaml
+++ b/loki/loki-headless-service.yaml
@@ -10,6 +10,11 @@ spec:
   ports:
   - name: gossip
     port: 7946
+    protocol: TCP
     targetPort: gossip
+  - name: grpc
+    port: 9096
+    protocol: TCP
+    targetPort: grpc
   selector:
-    app: loki
+    app.kubernetes.io/name: loki
diff --git a/loki/loki-ingest-service.yaml b/loki/loki-ingest-service.yaml
new file mode 100644
index 0000000..9bcc6e8
--- /dev/null
+++ b/loki/loki-ingest-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki-ingest
+  labels:
+    app.kubernetes.io/name: loki
+spec:
+  type: ClusterIP
+  ports:
+  - name: http
+    port: 3100
+    protocol: TCP
+    targetPort: http
+  selector:
+    app.kubernetes.io/name: loki
diff --git a/loki/loki-service.yaml b/loki/loki-query-service.yaml
similarity index 73%
rename from loki/loki-service.yaml
rename to loki/loki-query-service.yaml
index d71af69..99b5355 100644
--- a/loki/loki-service.yaml
+++ b/loki/loki-query-service.yaml
@@ -1,9 +1,9 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: loki
+  name: loki-query
   labels:
-    app: loki
+    app.kubernetes.io/name: loki
 spec:
   type: ClusterIP
   ports:
@@ -16,4 +16,4 @@ spec:
     protocol: TCP
     targetPort: grpc
   selector:
-    app: loki
+    app.kubernetes.io/name: loki
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index 4106326..81356ee 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -3,7 +3,7 @@ kind: StatefulSet
 metadata:
   name: loki
   labels:
-    app: loki
+    app.kubernetes.io/name: loki
 spec:
   replicas: 1 # should match the choice of CONFIG below
   podManagementPolicy: Parallel
@@ -11,12 +11,12 @@ spec:
    type: RollingUpdate
   selector:
     matchLabels:
-      app: loki
+      app.kubernetes.io/name: loki
   serviceName: loki-headless
   template:
     metadata:
       labels:
-        app: loki
+        app.kubernetes.io/name: loki
       annotations:
         prometheus.io/port: http
         prometheus.io/scrape: "true"
@@ -32,6 +32,11 @@ spec:
       ### for development
       terminationGracePeriodSeconds: 1
       shareProcessNamespace: true
+      hostAliases:
+      - ip: "127.0.0.1"
+        hostnames:
+        - "loki-ingest"
+        - "loki-query"
       containers:
       - name: debug
         image: curlimages/curl:7.78.0@sha256:a37d88a5b626208146c6d58f75b74a93c35dfc00b6b4676c4e9d2c166ea41954
@@ -39,12 +44,18 @@ spec:
         - /bin/sh
         - -ce
         - |
+          curl -o /tmp/logcli.zip -L https://github.com/grafana/loki/releases/download/v2.3.0/logcli-linux-amd64.zip
+          (cd /tmp; unzip logcli.zip; mv logcli-linux-amd64 logcli; chmod u+x logcli)
+          sleep 1; getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
+          curl --retry-connrefused --retry 10 -s http://${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:3100/config
           echo "Restart Loki using: kubectl -n $POD_NAMESPACE exec -c loki $POD_NAME -- pkill /usr/bin/loki"
           while true; do
             sleep 30
             getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
             curl -s http://localhost:3100/ready || true
-            curl -s http://localhost:3100/metrics | grep 'log_messages_total{' || true
+            curl -s http://localhost:3100/metrics | grep 'loki_ingester_chunk_stored_bytes_total{' || true
+            curl -s http://loki-query:3100/loki/api/v1/label/pod/values || true
+            /tmp/logcli query --no-labels '{container!=""} | logfmt' --limit=1 || true
           done
         env:
         - name: POD_NAME
@@ -57,13 +68,12 @@ spec:
               fieldPath: metadata.namespace
       ### for development end
       - name: loki
-        image: grafana/loki:2.3.0
+        image: docker.io/grafana/loki
         securityContext:
           readOnlyRootFilesystem: true
         args:
         - -config.file=/etc/loki/$(CONFIG).config.yaml
         - -config.expand-env=true
-        - -print-config-stderr=true
         #- -log.level=debug
         - -memberlist.transport-debug=true
         env:
diff --git a/loki/outputs-loki.conf b/loki/outputs-loki.conf
index 9602877..0d9e389 100644
--- a/loki/outputs-loki.conf
+++ b/loki/outputs-loki.conf
@@ -1,8 +1,11 @@
 [OUTPUT]
-    Name                   loki
-    Match                  *
-    Host                   loki
-    Port                   3100
-    Labels                 job=fluentbit
-    Label_Keys             $namespace, $pod_name, $pod_uid, $container_name
-    Auto_Kubernetes_Labels on
+    Name                    loki
+    Match                   *
+    Host                    loki-ingest
+    Port                    3100
+    # https://github.com/fluent/fluent-bit/issues/3269
+    # https://github.com/grafana/loki/issues/1544
+    labels                  job=fluentbit, namespace=$kubernetes['namespace_name'], pod=$kubernetes['pod_name'], container=$kubernetes['container_name']
+    remove_keys kubernetes,stream
+    #line_format json
+    log_level debug
diff --git a/loki/standalone.config.yaml b/loki/standalone.config.yaml
index cebf505..0a976e9 100644
--- a/loki/standalone.config.yaml
+++ b/loki/standalone.config.yaml
@@ -1,7 +1,7 @@
 # For more suggested overrides see https://github.com/grafana/loki/blob/v2.3.0/cmd/loki/loki-local-config.yaml
 # and of course https://github.com/grafana/loki/blob/v2.3.0/docs/sources/configuration/_index.md
 
-target: ingester,compactor,querier,query-frontend,table-manager
+#target: ingester,compactor,querier,query-frontend,table-manager
 
 auth_enabled: false
 
@@ -66,3 +66,9 @@ table_manager:
 #     kvstore:
 #       store: inmemory
 #   enable_api: true
+
+# frontend:
+#   downstream_url: ""
+
+# frontend_worker:
+#   frontend_address: ${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:9096

From c0b209e5814a0ea93c696f54df86bc1c93cf2f6a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 13 Aug 2021 21:46:37 +0200
Subject: [PATCH 35/49] Extracts debug behavior to a patch that can be
 commented out

---
 loki/debug-container.yaml   | 47 ++++++++++++++++++++++++++++++
 loki/kustomization.yaml     |  8 ++---
 loki/loki-statefulset.yaml  | 58 +++++--------------------------------
 loki/outputs-loki.conf      |  3 +-
 loki/standalone.config.yaml | 18 ------------
 5 files changed, 59 insertions(+), 75 deletions(-)
 create mode 100644 loki/debug-container.yaml

diff --git a/loki/debug-container.yaml b/loki/debug-container.yaml
new file mode 100644
index 0000000..c351284
--- /dev/null
+++ b/loki/debug-container.yaml
@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: loki
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 1
+      shareProcessNamespace: true
+      hostAliases:
+      - ip: "127.0.0.1"
+        hostnames:
+        - "loki-ingest"
+        - "loki-query"
+      containers:
+      - name: debug
+        image: curlimages/curl:7.78.0@sha256:a37d88a5b626208146c6d58f75b74a93c35dfc00b6b4676c4e9d2c166ea41954
+        command:
+        - /bin/sh
+        - -ce
+        - |
+          curl -o /tmp/logcli.zip -L https://github.com/grafana/loki/releases/download/v2.3.0/logcli-linux-amd64.zip
+          (cd /tmp; unzip logcli.zip; mv logcli-linux-amd64 logcli; chmod u+x logcli)
+          sleep 1; getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
+          curl --retry-connrefused --retry 10 -s http://${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:3100/config
+          echo "Restart Loki using: kubectl -n $POD_NAMESPACE exec -c loki $POD_NAME -- pkill /usr/bin/loki"
+          while true; do
+            sleep 30
+            getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
+            curl -s http://localhost:3100/ready || true
+            curl -s http://localhost:3100/metrics | grep 'loki_ingester_chunk_stored_bytes_total{' || true
+            curl -s http://loki-query:3100/loki/api/v1/label/pod/values || true
+            /tmp/logcli query --no-labels '{container!=""} | logfmt' --limit=1 || true
+          done
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      - name: loki
+        env:
+        - name: LOG_LEVEL
+          value: debug
diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index 4ceafc2..d84af42 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -15,10 +15,7 @@ resources:
 secretGenerator:
 - name: loki
   files:
-  - standalone.config.yaml
-  #- distributed-s3-minio.config.yaml
-  options:
-    disableNameSuffixHash: true
+  - loki.yaml=standalone.config.yaml
 
 configMapGenerator:
 - name: fluentbit
@@ -26,3 +23,6 @@ configMapGenerator:
   files:
   - outputs.conf=outputs-loki.conf
   - labelmap.json
+
+patchesStrategicMerge:
+- debug-container.yaml
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index 81356ee..195b671 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -5,10 +5,10 @@ metadata:
   labels:
     app.kubernetes.io/name: loki
 spec:
-  replicas: 1 # should match the choice of CONFIG below
+  replicas: 1
   podManagementPolicy: Parallel
   updateStrategy:
-   type: RollingUpdate
+    type: RollingUpdate
   selector:
     matchLabels:
       app.kubernetes.io/name: loki
@@ -21,64 +21,24 @@ spec:
         prometheus.io/port: http
         prometheus.io/scrape: "true"
     spec:
-      #serviceAccountName: loki
       securityContext:
         fsGroup: 10001
         runAsGroup: 10001
         runAsNonRoot: true
         runAsUser: 10001
-      #terminationGracePeriodSeconds: 25
-      #containers:
-      ### for development
-      terminationGracePeriodSeconds: 1
-      shareProcessNamespace: true
-      hostAliases:
-      - ip: "127.0.0.1"
-        hostnames:
-        - "loki-ingest"
-        - "loki-query"
+      terminationGracePeriodSeconds: 25
       containers:
-      - name: debug
-        image: curlimages/curl:7.78.0@sha256:a37d88a5b626208146c6d58f75b74a93c35dfc00b6b4676c4e9d2c166ea41954
-        command:
-        - /bin/sh
-        - -ce
-        - |
-          curl -o /tmp/logcli.zip -L https://github.com/grafana/loki/releases/download/v2.3.0/logcli-linux-amd64.zip
-          (cd /tmp; unzip logcli.zip; mv logcli-linux-amd64 logcli; chmod u+x logcli)
-          sleep 1; getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
-          curl --retry-connrefused --retry 10 -s http://${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:3100/config
-          echo "Restart Loki using: kubectl -n $POD_NAMESPACE exec -c loki $POD_NAME -- pkill /usr/bin/loki"
-          while true; do
-            sleep 30
-            getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
-            curl -s http://localhost:3100/ready || true
-            curl -s http://localhost:3100/metrics | grep 'loki_ingester_chunk_stored_bytes_total{' || true
-            curl -s http://loki-query:3100/loki/api/v1/label/pod/values || true
-            /tmp/logcli query --no-labels '{container!=""} | logfmt' --limit=1 || true
-          done
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-      ### for development end
       - name: loki
         image: docker.io/grafana/loki
         securityContext:
           readOnlyRootFilesystem: true
         args:
-        - -config.file=/etc/loki/$(CONFIG).config.yaml
+        - -config.file=/etc/loki/loki.yaml
         - -config.expand-env=true
-        #- -log.level=debug
-        - -memberlist.transport-debug=true
+        - -log.level=$(LOG_LEVEL)
         env:
-        - name: CONFIG
-          value: standalone
+        - name: LOG_LEVEL
+          value: info
         - name: POD_NAME
           valueFrom:
             fieldRef:
@@ -97,10 +57,6 @@ spec:
         - name: gossip
           containerPort: 7946
           protocol: TCP
-        # livenessProbe:
-        #   httpGet:
-        #     path: /ready
-        #     port: http
         readinessProbe:
           httpGet:
             path: /ready
diff --git a/loki/outputs-loki.conf b/loki/outputs-loki.conf
index 0d9e389..5f8f4f6 100644
--- a/loki/outputs-loki.conf
+++ b/loki/outputs-loki.conf
@@ -7,5 +7,4 @@
     # https://github.com/grafana/loki/issues/1544
     labels                  job=fluentbit, namespace=$kubernetes['namespace_name'], pod=$kubernetes['pod_name'], container=$kubernetes['container_name']
     remove_keys kubernetes,stream
-    #line_format json
-    log_level debug
+    log_level ${LOG_LEVEL}
diff --git a/loki/standalone.config.yaml b/loki/standalone.config.yaml
index 0a976e9..d995e67 100644
--- a/loki/standalone.config.yaml
+++ b/loki/standalone.config.yaml
@@ -1,8 +1,6 @@
 # For more suggested overrides see https://github.com/grafana/loki/blob/v2.3.0/cmd/loki/loki-local-config.yaml
 # and of course https://github.com/grafana/loki/blob/v2.3.0/docs/sources/configuration/_index.md
 
-#target: ingester,compactor,querier,query-frontend,table-manager
-
 auth_enabled: false
 
 server:
@@ -56,19 +54,3 @@ table_manager:
   retention_period: 0s
 
 # ruler:
-#   storage:
-#     type: local
-#     local:
-#       directory: /loki/rules
-#   rule_path: /loki/rules-temp
-#   alertmanager_url: http://localhost:9093
-#   ring:
-#     kvstore:
-#       store: inmemory
-#   enable_api: true
-
-# frontend:
-#   downstream_url: ""
-
-# frontend_worker:
-#   frontend_address: ${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:9096

From 63b006d67eb461080fb9cbef481bc33fa5c4e0ae Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 13 Aug 2021 22:02:53 +0200
Subject: [PATCH 36/49] Fix indentation

---
 loki/kustomization.yaml |  1 -
 loki/outputs-loki.conf  | 12 ++++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index d84af42..07132bf 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -22,7 +22,6 @@ configMapGenerator:
   behavior: merge
   files:
   - outputs.conf=outputs-loki.conf
-  - labelmap.json
 
 patchesStrategicMerge:
 - debug-container.yaml
diff --git a/loki/outputs-loki.conf b/loki/outputs-loki.conf
index 5f8f4f6..af11db8 100644
--- a/loki/outputs-loki.conf
+++ b/loki/outputs-loki.conf
@@ -1,10 +1,10 @@
 [OUTPUT]
-    Name                    loki
-    Match                   *
-    Host                    loki-ingest
-    Port                    3100
+    Name        loki
+    Match       *
+    Host        loki-ingest
+    Port        3100
     # https://github.com/fluent/fluent-bit/issues/3269
     # https://github.com/grafana/loki/issues/1544
-    labels                  job=fluentbit, namespace=$kubernetes['namespace_name'], pod=$kubernetes['pod_name'], container=$kubernetes['container_name']
+    labels      job=fluentbit, namespace=$kubernetes['namespace_name'], pod=$kubernetes['pod_name'], container=$kubernetes['container_name']
     remove_keys kubernetes,stream
-    log_level ${LOG_LEVEL}
+    log_level   ${LOG_LEVEL}

From cae23c9d1c3589d9a040c6c23d6200f81a41461f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 14 Aug 2021 06:32:37 +0200
Subject: [PATCH 37/49] Restores the official image's config path

---
 base/{fluentbit.conf => fluent-bit.conf} | 0
 base/fluentbit-daemonset.yaml            | 4 ++--
 base/kustomization.yaml                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename base/{fluentbit.conf => fluent-bit.conf} (100%)

diff --git a/base/fluentbit.conf b/base/fluent-bit.conf
similarity index 100%
rename from base/fluentbit.conf
rename to base/fluent-bit.conf
diff --git a/base/fluentbit-daemonset.yaml b/base/fluentbit-daemonset.yaml
index 6add83f..d86ed93 100644
--- a/base/fluentbit-daemonset.yaml
+++ b/base/fluentbit-daemonset.yaml
@@ -31,7 +31,7 @@ spec:
         command:
         - /fluent-bit/bin/fluent-bit
         - -c
-        - /etc/fluentbit/fluentbit.conf
+        - /fluent-bit/etc/fluent-bit.conf
         env:
         - name: LOG_LEVEL
           value: debug
@@ -55,7 +55,7 @@ spec:
         - name: db
           mountPath: /var/run/fluentbit
         - name: config
-          mountPath: /etc/fluentbit
+          mountPath: /fluent-bit/etc
         # - mountPath: /etc/machine-id
         #   name: etcmachineid
         #   readOnly: true
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index ed6e5cd..89d2813 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -17,7 +17,7 @@ resources:
 configMapGenerator:
 - name: fluentbit
   files:
-  - fluentbit.conf
+  - fluent-bit.conf
   - parser-cri.conf
   - input-kubernetes.conf
   - filter-kubernetes.conf

From 05dbf8db2fd3ebcc25eb41ccbf6c321dd68d0e8e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 14 Aug 2021 07:00:56 +0200
Subject: [PATCH 38/49] Using the go plugin we can send the .log record as log
 line

which helps human consumption when log records are json
---
 loki-plugin/fluentbit-loki-plugin.yaml | 17 +++++++++++++++++
 loki-plugin/kustomization.yaml         | 19 +++++++++++++++++++
 loki-plugin/labelmap.json              | 13 +++++++++++++
 loki-plugin/outputs-loki-plugin.conf   | 14 ++++++++++++++
 4 files changed, 63 insertions(+)
 create mode 100644 loki-plugin/fluentbit-loki-plugin.yaml
 create mode 100644 loki-plugin/kustomization.yaml
 create mode 100644 loki-plugin/labelmap.json
 create mode 100644 loki-plugin/outputs-loki-plugin.conf

diff --git a/loki-plugin/fluentbit-loki-plugin.yaml b/loki-plugin/fluentbit-loki-plugin.yaml
new file mode 100644
index 0000000..630ad69
--- /dev/null
+++ b/loki-plugin/fluentbit-loki-plugin.yaml
@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentbit
+spec:
+  template:
+    spec:
+      containers:
+      - name: fluentbit
+        # https://github.com/grafana/loki/tree/main/clients/cmd/fluent-bit#building
+        image: docker.io/solsson/fluentbit-loki
+        command:
+        - /fluent-bit/bin/fluent-bit
+        - -c
+        - /fluent-bit/etc/fluent-bit.conf
+        - -e
+        - /fluent-bit/bin/out_grafana_loki.so
diff --git a/loki-plugin/kustomization.yaml b/loki-plugin/kustomization.yaml
new file mode 100644
index 0000000..4406e81
--- /dev/null
+++ b/loki-plugin/kustomization.yaml
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../loki
+
+images:
+- name: docker.io/solsson/fluentbit-loki
+  newTag: cd467d7a00610ec4ef01da72427693e32157ce44
+
+patchesStrategicMerge:
+- fluentbit-loki-plugin.yaml
+
+configMapGenerator:
+- name: fluentbit
+  behavior: merge
+  files:
+  - outputs.conf=outputs-loki-plugin.conf
+  - labelmap.json
diff --git a/loki-plugin/labelmap.json b/loki-plugin/labelmap.json
new file mode 100644
index 0000000..b616ee0
--- /dev/null
+++ b/loki-plugin/labelmap.json
@@ -0,0 +1,13 @@
+{
+  "kubernetes": {
+    "container_name": "container",
+    "host": "node",
+    "namespace_name": "namespace",
+    "pod_name": "pod",
+    "labels" : {
+      "app.kubernetes.io/name": "app",
+      "app": "app"
+    }
+  },
+  "stream": "stream"
+}
diff --git a/loki-plugin/outputs-loki-plugin.conf b/loki-plugin/outputs-loki-plugin.conf
new file mode 100644
index 0000000..a8997a4
--- /dev/null
+++ b/loki-plugin/outputs-loki-plugin.conf
@@ -0,0 +1,14 @@
+
+# https://grafana.com/docs/loki/latest/clients/fluentbit/#configuration-options
+[OUTPUT]
+    Name            grafana-loki
+    Match           *
+    LogLevel        ${LOG_LEVEL}
+    Url             http://loki-ingest:3100/loki/api/v1/push
+
+    DqueDir         /var/run/fluent-bit/loki-plugin
+
+    Labels          {job="fluentbit"}
+    LabelMapPath    /fluent-bit/etc/labelmap.json
+    DropSingleKey   true
+    RemoveKeys      logtag,stream,kubernetes

From f130d8df8ecb94383362c44eb3caa5edc7e91602 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 14 Aug 2021 07:07:34 +0200
Subject: [PATCH 39/49] Grafana's build is also based on the fluent-bit 1.8.3
 image

---
 loki-plugin/distributed.config.yaml    | 56 ++++++++++++++++++++++++++
 loki-plugin/fluentbit-loki-plugin.yaml |  3 +-
 loki-plugin/kustomization.yaml         |  4 +-
 loki/debug-container.yaml              |  2 +-
 4 files changed, 60 insertions(+), 5 deletions(-)
 create mode 100644 loki-plugin/distributed.config.yaml

diff --git a/loki-plugin/distributed.config.yaml b/loki-plugin/distributed.config.yaml
new file mode 100644
index 0000000..d995e67
--- /dev/null
+++ b/loki-plugin/distributed.config.yaml
@@ -0,0 +1,56 @@
+# For more suggested overrides see https://github.com/grafana/loki/blob/v2.3.0/cmd/loki/loki-local-config.yaml
+# and of course https://github.com/grafana/loki/blob/v2.3.0/docs/sources/configuration/_index.md
+
+auth_enabled: false
+
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  lifecycler:
+    address: 0.0.0.0
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  wal:
+    enabled: true
+    dir: /loki/wal
+  max_transfer_retries: 0
+
+schema_config:
+  configs:
+  - from: 2021-08-13
+    store: boltdb-shipper
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/boltdb-shipper-active
+    cache_location: /loki/boltdb-shipper-cache
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 1h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+# ruler:
diff --git a/loki-plugin/fluentbit-loki-plugin.yaml b/loki-plugin/fluentbit-loki-plugin.yaml
index 630ad69..77342e3 100644
--- a/loki-plugin/fluentbit-loki-plugin.yaml
+++ b/loki-plugin/fluentbit-loki-plugin.yaml
@@ -7,8 +7,7 @@ spec:
     spec:
       containers:
       - name: fluentbit
-        # https://github.com/grafana/loki/tree/main/clients/cmd/fluent-bit#building
-        image: docker.io/solsson/fluentbit-loki
+        image: docker.io/grafana/fluent-bit-plugin-loki
         command:
         - /fluent-bit/bin/fluent-bit
         - -c
diff --git a/loki-plugin/kustomization.yaml b/loki-plugin/kustomization.yaml
index 4406e81..db19126 100644
--- a/loki-plugin/kustomization.yaml
+++ b/loki-plugin/kustomization.yaml
@@ -5,8 +5,8 @@ resources:
 - ../loki
 
 images:
-- name: docker.io/solsson/fluentbit-loki
-  newTag: cd467d7a00610ec4ef01da72427693e32157ce44
+- name: docker.io/grafana/fluent-bit-plugin-loki
+  newTag: latest@sha256:c5b84f0a41359c6dabdc27e30797157c82e5f440001723e5a6ae0fa255b12811
 
 patchesStrategicMerge:
 - fluentbit-loki-plugin.yaml
diff --git a/loki/debug-container.yaml b/loki/debug-container.yaml
index c351284..24adff1 100644
--- a/loki/debug-container.yaml
+++ b/loki/debug-container.yaml
@@ -30,7 +30,7 @@ spec:
             curl -s http://localhost:3100/ready || true
             curl -s http://localhost:3100/metrics | grep 'loki_ingester_chunk_stored_bytes_total{' || true
             curl -s http://loki-query:3100/loki/api/v1/label/pod/values || true
-            /tmp/logcli query --no-labels '{container!=""} | logfmt' --limit=1 || true
+            /tmp/logcli query --no-labels '{container!=""}' --limit=1 || true
           done
         env:
         - name: POD_NAME

From de8c0b2d4ff55930752a916466101a34979b2832 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 14 Aug 2021 16:51:35 +0200
Subject: [PATCH 40/49] A loki test/examples job that can be applied with -k or
 -f

---
 loki/debug-container.yaml   | 47 -------------------------------------
 loki/kustomization.yaml     |  3 ---
 loki/test/Kustomization     |  3 +++
 loki/test/lokitest-job.yaml | 40 +++++++++++++++++++++++++++++++
 4 files changed, 43 insertions(+), 50 deletions(-)
 delete mode 100644 loki/debug-container.yaml
 create mode 100644 loki/test/Kustomization
 create mode 100644 loki/test/lokitest-job.yaml

diff --git a/loki/debug-container.yaml b/loki/debug-container.yaml
deleted file mode 100644
index 24adff1..0000000
--- a/loki/debug-container.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: loki
-spec:
-  template:
-    spec:
-      terminationGracePeriodSeconds: 1
-      shareProcessNamespace: true
-      hostAliases:
-      - ip: "127.0.0.1"
-        hostnames:
-        - "loki-ingest"
-        - "loki-query"
-      containers:
-      - name: debug
-        image: curlimages/curl:7.78.0@sha256:a37d88a5b626208146c6d58f75b74a93c35dfc00b6b4676c4e9d2c166ea41954
-        command:
-        - /bin/sh
-        - -ce
-        - |
-          curl -o /tmp/logcli.zip -L https://github.com/grafana/loki/releases/download/v2.3.0/logcli-linux-amd64.zip
-          (cd /tmp; unzip logcli.zip; mv logcli-linux-amd64 logcli; chmod u+x logcli)
-          sleep 1; getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
-          curl --retry-connrefused --retry 10 -s http://${POD_NAME}.loki-headless.${POD_NAMESPACE}.svc.cluster.local:3100/config
-          echo "Restart Loki using: kubectl -n $POD_NAMESPACE exec -c loki $POD_NAME -- pkill /usr/bin/loki"
-          while true; do
-            sleep 30
-            getent hosts _gossip._tcp.loki-headless.${POD_NAMESPACE}.svc.cluster.local || true
-            curl -s http://localhost:3100/ready || true
-            curl -s http://localhost:3100/metrics | grep 'loki_ingester_chunk_stored_bytes_total{' || true
-            curl -s http://loki-query:3100/loki/api/v1/label/pod/values || true
-            /tmp/logcli query --no-labels '{container!=""}' --limit=1 || true
-          done
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-      - name: loki
-        env:
-        - name: LOG_LEVEL
-          value: debug
diff --git a/loki/kustomization.yaml b/loki/kustomization.yaml
index 07132bf..0326f89 100644
--- a/loki/kustomization.yaml
+++ b/loki/kustomization.yaml
@@ -22,6 +22,3 @@ configMapGenerator:
   behavior: merge
   files:
   - outputs.conf=outputs-loki.conf
-
-patchesStrategicMerge:
-- debug-container.yaml
diff --git a/loki/test/Kustomization b/loki/test/Kustomization
new file mode 100644
index 0000000..4436eb4
--- /dev/null
+++ b/loki/test/Kustomization
@@ -0,0 +1,3 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources: ["lokitest-job.yaml"]
diff --git a/loki/test/lokitest-job.yaml b/loki/test/lokitest-job.yaml
new file mode 100644
index 0000000..5b0e651
--- /dev/null
+++ b/loki/test/lokitest-job.yaml
@@ -0,0 +1,40 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: lokitest
+spec:
+  backoffLimit: 5
+  template:
+    spec:
+      containers:
+      - name: lokitest
+        image: yolean/toil:16986b1488a60eaf2d81a8ff8f8f54f3aeacad1a@sha256:4717a46e082ac5a14da9a3e6e2aa2cd012f94989b2ee3c41ce7d39fa463d9915
+        command:
+        - /bin/bash
+        - -cex
+        - |
+          alias curl='curl -f -s --retry-connrefused --retry 20'
+          curl http://loki-ingest:3100/ready
+          curl http://loki-ingest:3100/config
+          curl http://loki-query:3100/loki/api/v1/labels
+          curl http://loki-query:3100/loki/api/v1/label/pod/values
+          function stream1 {
+            set +x
+            log=$1; [ -n "$log" ] || log="Logged at $(date -Ins)"
+            echo '{"streams":[{"stream":{"job":"lokitest"},"values":[["'$(date +%s%N)'","'$log'"]]}]}'
+            set -x
+          }
+          stream1 | jq -c .
+          t=$(date +%F'T'%T.%N'Z')
+          curl -H "Content-Type: application/json" -XPOST -w "%{http_code}\n" "http://loki-ingest:3100/loki/api/v1/push" --data-raw "$(stream1)"
+          export LOKI_ADDR=http://loki-query:3100
+          logcli query '{job="lokitest"}' --from=$t
+          # Ingestion might work while distribution doesn't, in which case the results differ between pods (TODO after a delay?)
+          for POD_IP in $(getent hosts loki-headless | cut -d' ' -f1); do
+            echo "# Loki pod IP $POD_IP"
+            curl -H "Content-Type: application/json" -XPOST -w "%{http_code}\n" "http://$POD_IP:3100/loki/api/v1/push" --data-raw "$(stream1)"
+            logcli --addr=http://$POD_IP:3100 query '{job="lokitest"}' --from=$t
+          done
+          echo "Sleeping for experiments through exec"
+          sleep 3600
+      restartPolicy: Never

From 614ea0a3897e51bf26f6a9e16e11c9056150ebd3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 14 Aug 2021 19:26:56 +0200
Subject: [PATCH 41/49] Was meant for the kustomize-base-loki-distributed
 branch

---
 loki-plugin/distributed.config.yaml | 56 -----------------------------
 1 file changed, 56 deletions(-)
 delete mode 100644 loki-plugin/distributed.config.yaml

diff --git a/loki-plugin/distributed.config.yaml b/loki-plugin/distributed.config.yaml
deleted file mode 100644
index d995e67..0000000
--- a/loki-plugin/distributed.config.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
-# For more suggested overrides see https://github.com/grafana/loki/blob/v2.3.0/cmd/loki/loki-local-config.yaml
-# and of course https://github.com/grafana/loki/blob/v2.3.0/docs/sources/configuration/_index.md
-
-auth_enabled: false
-
-server:
-  http_listen_address: 0.0.0.0
-  http_listen_port: 3100
-  grpc_listen_port: 9096
-
-ingester:
-  lifecycler:
-    address: 0.0.0.0
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-  wal:
-    enabled: true
-    dir: /loki/wal
-  max_transfer_retries: 0
-
-schema_config:
-  configs:
-  - from: 2021-08-13
-    store: boltdb-shipper
-    object_store: filesystem
-    schema: v11
-    index:
-      prefix: index_
-      period: 24h
-
-storage_config:
-  boltdb_shipper:
-    active_index_directory: /loki/boltdb-shipper-active
-    cache_location: /loki/boltdb-shipper-cache
-    shared_store: filesystem
-  filesystem:
-    directory: /loki/chunks
-
-compactor:
-  working_directory: /loki/boltdb-shipper-compactor
-  shared_store: filesystem
-
-limits_config:
-  reject_old_samples: true
-  reject_old_samples_max_age: 1h
-
-chunk_store_config:
-  max_look_back_period: 0s
-
-table_manager:
-  retention_deletes_enabled: false
-  retention_period: 0s
-
-# ruler:

From d7805fb423b0700dfbc0ff4e3c29f0fdee9a2213 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 06:50:17 +0200
Subject: [PATCH 42/49] Use the exclusion annotation for anything but promtail
 itself

---
 base/filter-kubernetes.conf | 4 ++--
 base/input-kubernetes.conf  | 3 +--
 loki/loki-statefulset.yaml  | 1 +
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/base/filter-kubernetes.conf b/base/filter-kubernetes.conf
index 33d62be..642586d 100644
--- a/base/filter-kubernetes.conf
+++ b/base/filter-kubernetes.conf
@@ -6,6 +6,6 @@
     Regex_Parser        kube-tag
     # Merge_Log           On
     # Merge_Log_Key       log_processed
-    # K8S-Logging.Parser  On
-    # K8S-Logging.Exclude Off
+    K8S-Logging.Parser  On
+    K8S-Logging.Exclude On
     Annotations         Off
diff --git a/base/input-kubernetes.conf b/base/input-kubernetes.conf
index ca486a5..d457ae3 100644
--- a/base/input-kubernetes.conf
+++ b/base/input-kubernetes.conf
@@ -5,8 +5,7 @@
     Skip_Long_Lines   On
     DB.locking        true
     Path              /var/log/pods/*/*/*.log
-    # We could exclude loki using K8S-Logging.Exclude bit for fluentbit we want to avoid loops regardless of filters
-    Exclude_Path      /var/log/pods/*/fluentbit/*.log,/var/log/pods/*/loki/*.log
+    Exclude_Path      /var/log/pods/*/fluentbit/*.log
     Parser            cri
     Tag               kube.<pod_id>.<namespace_name>.<pod_name>.<container_name>
     Tag_Regex         /var/log/pods/(?<namespace_name>[^_/]+)_(?<pod_name>[^_/]+)_(?<pod_id>[^_/]+)/(?<container_name>[^/]+)/.*
diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index 195b671..f89dffc 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -20,6 +20,7 @@ spec:
       annotations:
         prometheus.io/port: http
         prometheus.io/scrape: "true"
+        fluentbit.io/exclude: "true"
     spec:
       securityContext:
         fsGroup: 10001

From 79771fb8f6fea41f80fb7415c17152c9691ffc6f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 06:51:16 +0200
Subject: [PATCH 43/49] Tests can fail explicitly with exit 1 where desired

---
 loki/test/lokitest-job.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/loki/test/lokitest-job.yaml b/loki/test/lokitest-job.yaml
index 5b0e651..2b878e3 100644
--- a/loki/test/lokitest-job.yaml
+++ b/loki/test/lokitest-job.yaml
@@ -11,7 +11,7 @@ spec:
         image: yolean/toil:16986b1488a60eaf2d81a8ff8f8f54f3aeacad1a@sha256:4717a46e082ac5a14da9a3e6e2aa2cd012f94989b2ee3c41ce7d39fa463d9915
         command:
         - /bin/bash
-        - -cex
+        - -cx
         - |
           alias curl='curl -f -s --retry-connrefused --retry 20'
           curl http://loki-ingest:3100/ready

From a81086ccfed5d4fb24253becea39b5cb30333ef8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 10:18:09 +0200
Subject: [PATCH 44/49] Adds metrics and exec example for the test job

---
 loki/test/lokitest-job.yaml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/loki/test/lokitest-job.yaml b/loki/test/lokitest-job.yaml
index 2b878e3..c732fad 100644
--- a/loki/test/lokitest-job.yaml
+++ b/loki/test/lokitest-job.yaml
@@ -9,12 +9,22 @@ spec:
       containers:
       - name: lokitest
         image: yolean/toil:16986b1488a60eaf2d81a8ff8f8f54f3aeacad1a@sha256:4717a46e082ac5a14da9a3e6e2aa2cd012f94989b2ee3c41ce7d39fa463d9915
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         command:
         - /bin/bash
         - -cx
         - |
           alias curl='curl -f -s --retry-connrefused --retry 20'
           curl http://loki-ingest:3100/ready
+          curl http://loki-ingest:3100/metrics | grep memberlist | grep _count | grep -v gauge
           curl http://loki-ingest:3100/config
           curl http://loki-query:3100/loki/api/v1/labels
           curl http://loki-query:3100/loki/api/v1/label/pod/values
@@ -35,6 +45,6 @@ spec:
             curl -H "Content-Type: application/json" -XPOST -w "%{http_code}\n" "http://$POD_IP:3100/loki/api/v1/push" --data-raw "$(stream1)"
             logcli --addr=http://$POD_IP:3100 query '{job="lokitest"}' --from=$t
           done
-          echo "Sleeping for experiments through exec"
+          echo "Sleeping for experiments through: kubectl -n $POD_NAMESPACE exec $POD_NAME -ti -- bash"
           sleep 3600
       restartPolicy: Never

From 3426a373f463091f517605bb9d78215eab9a2645 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 11:27:51 +0200
Subject: [PATCH 45/49] Getting error "TCPTransport: unknown message type"
 msgType=G

from tcp_transport.go:313

Use PodMonitor etc. instead.
---
 loki/loki-statefulset.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/loki/loki-statefulset.yaml b/loki/loki-statefulset.yaml
index f89dffc..e01f6ab 100644
--- a/loki/loki-statefulset.yaml
+++ b/loki/loki-statefulset.yaml
@@ -18,8 +18,6 @@ spec:
       labels:
         app.kubernetes.io/name: loki
       annotations:
-        prometheus.io/port: http
-        prometheus.io/scrape: "true"
         fluentbit.io/exclude: "true"
     spec:
       securityContext:

From a77aa023fc1851c1cb6a7231efa443fe4397bdb1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 14:37:54 +0200
Subject: [PATCH 46/49] I can't see any effect of the "wildcard" toleration

---
 base/fluentbit-daemonset.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/base/fluentbit-daemonset.yaml b/base/fluentbit-daemonset.yaml
index d86ed93..dabea98 100644
--- a/base/fluentbit-daemonset.yaml
+++ b/base/fluentbit-daemonset.yaml
@@ -23,8 +23,6 @@ spec:
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
-      - operator: Exists
-        effect: NoExecute
       containers:
       - name: fluentbit
         image: docker.io/fluent/fluent-bit

From 3f7e32b0afa7d3acc959765563eb97ab8e0d8a55 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 14:38:42 +0200
Subject: [PATCH 47/49] Removes the old Kafka REST example

---
 fluent-bit-config-kafka-rest.yml    | 86 -----------------------------
 fluent-bit-daemonset-kafka-rest.yml | 65 ----------------------
 testpod.yml                         | 22 --------
 3 files changed, 173 deletions(-)
 delete mode 100644 fluent-bit-config-kafka-rest.yml
 delete mode 100644 fluent-bit-daemonset-kafka-rest.yml
 delete mode 100644 testpod.yml

diff --git a/fluent-bit-config-kafka-rest.yml b/fluent-bit-config-kafka-rest.yml
deleted file mode 100644
index 606c408..0000000
--- a/fluent-bit-config-kafka-rest.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-kind: ConfigMap
-metadata:
-  name: fluent-bit-config-kafka-rest
-  namespace: kube-system
-apiVersion: v1
-data:
-  fluent-bit.conf: |-
-    [SERVICE]
-        Flush          1
-        Daemon         Off
-        Log_Level      info
-        Parsers_File   parsers.conf
-
-    [INPUT]
-        Name           tail
-        Tag            kube.*
-        Path           /var/log/containers/*.log
-        Parser         docker
-        DB             /var/log/flb_kube.db
-        Mem_Buf_Limit  5MB
-
-    [FILTER]
-        Name           kubernetes
-        Match          kube.*
-        Kube_URL       https://kubernetes.default.svc:443
-        Merge_JSON_Log On
-
-    [OUTPUT]
-        Name            kafka-rest
-        Match           *
-        Host            ${KAFKA_REST_HOST}
-        Port            ${KAFKA_REST_PORT}
-        Time_Key        @timestamp
-        Tag_Key         _fluent-tag
-        Include_Tag_Key On
-        Tag_Key         My_Tag_Key
-        Topic           ${KAFKA_TOPIC}
-        # Partition     0
-        # Message_Key abc
-
-  parsers.conf: |-
-    [PARSER]
-        Name   apache
-        Format regex
-        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   apache2
-        Format regex
-        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   apache_error
-        Format regex
-        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
-
-    [PARSER]
-        Name   nginx
-        Format regex
-        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   json-test
-        Format json
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name        docker
-        Format      json
-        Time_Key    time
-        Time_Format %Y-%m-%dT%H:%M:%S.%L
-        Time_Keep   On
-
-    [PARSER]
-        Name        syslog
-        Format      regex
-        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
-        Time_Key    time
-        Time_Format %b %d %H:%M:%S
diff --git a/fluent-bit-daemonset-kafka-rest.yml b/fluent-bit-daemonset-kafka-rest.yml
deleted file mode 100644
index 0c431d7..0000000
--- a/fluent-bit-daemonset-kafka-rest.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: fluent-bit
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: fluent-bit-logging
-    app.kubernetes.io/version: v1
-    kubernetes.io/cluster-service: "true"
-spec:
-  selector:
-    matchLabels:
-      k8s-app: fluent-bit-logging
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: fluent-bit-logging
-        app.kubernetes.io/version: v1
-        kubernetes.io/cluster-service: "true"
-    spec:
-      containers:
-      - name: fluent-bit
-        image: fluent/fluent-bit:0.12.19
-        env:
-          - name:  KAFKA_REST_HOST
-            value: rest.kafka.svc.cluster.local
-          - name:  KAFKA_REST_PORT
-            value: "80"
-          - name:  KAFKA_TOPIC
-            value: fluent-bit
-        command:
-          - /fluent-bit/bin/fluent-bit
-          - -c
-          - /fluent-bit/etc/fluent-bit.conf
-        resources:
-          limits:
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 100Mi
-        volumeMounts:
-        - name: config
-          mountPath: /fluent-bit/etc
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
-          readOnly: true
-        - name: minikube
-          mountPath: /mnt/sda1/var/lib/docker/containers
-          readOnly: true
-      terminationGracePeriodSeconds: 10
-      volumes:
-      - name: config
-        configMap:
-          name: fluent-bit-config-kafka-rest
-      - name: varlog
-        hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
-        hostPath:
-          path: /var/lib/docker/containers
-      - name: minikube
-        hostPath:
-          path: /mnt/sda1/var/lib/docker/containers
diff --git a/testpod.yml b/testpod.yml
deleted file mode 100644
index 9ee3cd0..0000000
--- a/testpod.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: testpod-fluentbit-kafka
-  namespace: default
-spec:
-  containers:
-  - name: kafkacat
-    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
-    command:
-      - bash
-      - -ec
-      - >
-        cd /usr/local/bin;
-        echo '#!/bin/bash' > logs-cat;
-        chmod +x logs-cat;
-        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -C -t fluent-bit -o end" >> logs-cat;
-
-        echo "Won't cat logs to stdout because it would create an infinite loop of log aggregation.";
-        echo "Instead run something like:";
-        echo "kubectl -n default exec -ti testpod-fluentbit-kafka -- logs-cat";
-        tail -f /dev/null;

From adab693f70339c9a71e2d843deb41078df4fe34d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 14:59:13 +0200
Subject: [PATCH 48/49] We could add more base folders to restore Elasticsearch
 and Kafka

examples but I think they are two extremes where the former requires
custom parsing to be fully useful and the latter makes no case for
the Fluentbit data pipeline at all without a consuming step.
---
 README.md | 70 +++++++++++++------------------------------------------
 1 file changed, 16 insertions(+), 54 deletions(-)

diff --git a/README.md b/README.md
index 9d4460e..aa9db5b 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,5 @@
 # Kubernetes Logging with Fluent Bit
 
-
-
 [Fluent Bit](http://fluentbit.io) is a lightweight and extensible __Log Processor__ that comes with full support for Kubernetes:
 
 - Read Kubernetes/Docker log files from the file system or through systemd Journal
@@ -16,72 +14,36 @@ This repository contains a set of Yaml files to deploy Fluent Bit which consider
 
 ```
 $ kubectl create namespace logging
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-service-account.yaml
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-role.yaml
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-role-binding.yaml
+$ kubectl apply -k ./base
 ```
 
+(`./base` can be replaced with a URL if [#90](https://github.com/fluent/fluent-bit-kubernetes-logging/pull/90) gets merged)
+
 If you are deploying fluent-bit on openshift, you additionally need to run:
 
 ```
 $ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/fluent-bit-openshift-security-context-constraints.yaml
 ```
 
-#### Fluent Bit to Elasticsearch
-
-The next step is to create a ConfigMap that will be used by our Fluent Bit DaemonSet:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-configmap.yaml
-```
-
-If the cluster uses a CRI runtime, like containerd or CRI-O, change the `Parser` described in `input-kubernetes.conf` from docker to cri.
-
-Fluent Bit DaemonSet ready to be used with Elasticsearch on a normal Kubernetes Cluster:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds.yaml
-```
-
-#### Fluent Bit to Elasticsearch on Minikube
-
-If you are using Minikube for testing purposes, use the following alternative DaemonSet manifest:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/elasticsearch/fluent-bit-ds-minikube.yaml
-```
-
-#### Fluent Bit to Kafka
-
-Create a ConfigMap that will be used by our Fluent Bit DaemonSet:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-configmap.yaml
-```
-
-Fluent Bit DaemonSet ready to be used with Kafka on a normal Kubernetes Cluster:
+The base only configures [Stdout](https://docs.fluentbit.io/manual/pipeline/outputs/standard-output) output.
+Log records are found using `kubectl logs` on fluentbit pods.
+This is a good way to experiment with configuration.
+Switch to [Format](https://docs.fluentbit.io/manual/pipeline/outputs/standard-output#configuration-parameters) `msgpack` to also see the "tag" (helpful when developing your pipeline).
 
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds.yaml
-```
+## Logs ingested to Loki
 
-#### Fluent Bit to Kafka on Minikube
+This repository serves as example of Kubernetes yaml for a log processing stack,
+not as example of [how](https://docs.fluentbit.io/manual/concepts/data-pipeline) Fluent-bit can process, refine and forward log entries.
 
-If you are using Minikube for testing purposes, use the following alternative DaemonSet manifest:
-
-```
-$ kubectl create -f https://raw.githubusercontent.com/fluent/fluent-bit-kubernetes-logging/master/output/kafka/fluent-bit-ds-minikube.yaml
-```
+Use the [loki](./loki) base to replace Stdout with forwarding to a standalone Loki instance. Loki can do some processing at [query time](https://grafana.com/docs/loki/latest/logql/) which helps keep this repo light on pipeline config.
 
-## Details
+The [loki-plugin](./loki-plugin) base uses an out-of-tree plugin with `DropSingleKey`
+so that the log records forwarded are the actual logged lines from pods,
+but still queryable on labels like `pod` and `container`.
 
-The default configuration of Fluent Bit makes sure of the following:
+See the [lokitest](./loki/test/lokitest-job.yaml) Job for some examples of how to consume logs through Loki.
 
-- Consume all containers logs from the running Node.
-- The [Tail input plugin](http://fluentbit.io/documentation/0.12/input/tail.html) will not append more than __5MB__  into the engine until they are flushed to the Elasticsearch backend. This limit aims to provide a workaround for [backpressure](http://fluentbit.io/documentation/0.13/configuration/backpressure.html) scenarios.
-- The Kubernetes filter will enrich the logs with Kubernetes metadata, specifically _labels_ and _annotations_. The filter only goes to the API Server when it cannot find the cached info, otherwise it uses the cache.
-- The default backend in the configuration is Elasticsearch set by the [Elasticsearch Output Plugin](http://fluentbit.io/documentation/0.13/output/elasticsearch.html). It uses the Logstash format to ingest the logs. If you need a different Index and Type, please refer to the plugin option and do your own adjustments.
-- There is an option called __Retry_Limit__ set to False that means if Fluent Bit cannot flush the records to Elasticsearch it will re-try indefinitely until it succeeds.
+The loki folders also serve as [example](./loki/kustomization.yaml) of how to use Kustomize to override selected parts of the `base`'s *.conf.
 
 ## Get in touch with us!
 

From c1a93cbb2a8df34ecf3c20798528c0289cdf3003 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 15 Aug 2021 15:13:05 +0200
Subject: [PATCH 49/49] When distribution works there's no need for a wait
 before query

About the "toil" image, grafana/logcli doesn't have curl and I
wanted to provide an example of how to consume logs from Loki.

We echo out the logcli commands instead and do everything with curlimages/curl
---
 loki/test/lokitest-job.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/loki/test/lokitest-job.yaml b/loki/test/lokitest-job.yaml
index c732fad..7dc8d76 100644
--- a/loki/test/lokitest-job.yaml
+++ b/loki/test/lokitest-job.yaml
@@ -8,6 +8,7 @@ spec:
     spec:
       containers:
       - name: lokitest
+        # Use any image with curl and logcli, or use curlimages/curl and replace logcli use with REST api calls
         image: yolean/toil:16986b1488a60eaf2d81a8ff8f8f54f3aeacad1a@sha256:4717a46e082ac5a14da9a3e6e2aa2cd012f94989b2ee3c41ce7d39fa463d9915
         env:
         - name: POD_NAME
@@ -39,7 +40,7 @@ spec:
           curl -H "Content-Type: application/json" -XPOST -w "%{http_code}\n" "http://loki-ingest:3100/loki/api/v1/push" --data-raw "$(stream1)"
           export LOKI_ADDR=http://loki-query:3100
           logcli query '{job="lokitest"}' --from=$t
-          # Ingestion might work while distribution doesn't, in which case the results differ between pods (TODO after a delay?)
+          # Test distriuted Loki, where ingestion might work while distribution doesn't; check if results differ between pods
           for POD_IP in $(getent hosts loki-headless | cut -d' ' -f1); do
             echo "# Loki pod IP $POD_IP"
             curl -H "Content-Type: application/json" -XPOST -w "%{http_code}\n" "http://$POD_IP:3100/loki/api/v1/push" --data-raw "$(stream1)"