From 5ee74e5a20c3e9d632ce04d5a1606b3bf77c6e5c Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Thu, 28 Sep 2023 14:45:50 +0200
Subject: [PATCH 01/26] Remove redundant restart

We don't need to restart Elasticseach after this task. Everything is set
in a similar task earlier. This one is only to change the start
bevaviour to a safer one (not reinitializing the cluster). The change is
only needed during restarts, so whenever Elasticsearch is restarted, the
new version will be used.

fixes #278
---
 roles/elasticsearch/tasks/main.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 4b813117..c6d9bd59 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -237,8 +237,6 @@
     group: root
     mode: 0644
     backup: "{{ elasticsearch_config_backup }}"
-  notify:
-    - Restart Elasticsearch
   when: elasticsearch_manage_yaml | bool
 
 - name: Show Info about heap

From a759794a2b5c46f8ceb76a7c5a5d9b20d7f8d5f7 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Thu, 28 Sep 2023 15:24:35 +0200
Subject: [PATCH 02/26] Only restart Elasticsearch if it was already running

---
 .../tasks/elasticsearch-security.yml          | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 0b1a95ac..a22de703 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -352,25 +352,30 @@
     name: elasticsearch
     state: started
     enabled: yes
+  register: elasticsearch_freshstart
 
 - name: Wait for all instances to start
   ansible.builtin.include_tasks: wait_for_instance.yml
   loop: "{{ groups['elasticsearch'] }}"
 
-- name: Force all notified handlers to run at this point, not waiting for normal sync points
-  ansible.builtin.meta: flush_handlers
-  tags:
-    - certificates
-    - renew_ca
-    - renew_es_cert
-
-- name: Wait for all instances to start
-  ansible.builtin.include_tasks: wait_for_instance.yml
-  loop: "{{ groups['elasticsearch'] }}"
-  tags:
-    - certificates
-    - renew_ca
-    - renew_es_cert
+- name: Restart if Elasticsearch was already running
+  when:
+    - not elasticsearch_freshstart.changed | bool
+  block:
+    - name: Force all notified handlers to run at this point, not waiting for normal sync points
+      ansible.builtin.meta: flush_handlers
+      tags:
+        - certificates
+        - renew_ca
+        - renew_es_cert
+
+    - name: Wait for all instances to start
+      ansible.builtin.include_tasks: wait_for_instance.yml
+      loop: "{{ groups['elasticsearch'] }}"
+      tags:
+        - certificates
+        - renew_ca
+        - renew_es_cert
 
 - name: Check for passwords being set
   ansible.builtin.stat:

From 099024d01ba3f068891b549a69fc2e9fa1b2dcc9 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Thu, 28 Sep 2023 15:38:34 +0200
Subject: [PATCH 03/26] Check API twice

once for availability, once for data
---
 .../tasks/elasticsearch-security.yml          | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index a22de703..cc6e363c 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -388,6 +388,26 @@
     elasticsearch_http_protocol: "https"
   when: elasticsearch_http_security
 
+- name: Check for cluster status with bootstrap password - no fail
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
+    user: elastic
+    password: "{{ elasticsearch_bootstrap_pw }}"
+    validate_certs: false
+  register: elasticsearch_cluster_status_bootstrap
+  changed_when: false
+  failed_when: false
+  no_log: "{{ elasticstack_no_log }}"
+  when:
+    - not elasticsearch_passwords_file.stat.exists | bool
+    - groups['elasticsearch'] | length > 1
+  until: elasticsearch_cluster_status_bootstrap.json.status == "green"
+  retries: 5
+  delay: 10
+
+# We need this check twice. One with failed_when: false to wait for the API to be actually available. And a second time to
+# check the actual return code. Should not cause a huge delay.
+
 - name: Check for cluster status with bootstrap password
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
@@ -415,6 +435,26 @@
   delegate_to: "{{ elasticstack_ca }}"
   when: elasticsearch_passwords_file.stat.exists | bool
 
+- name: Check for cluster status with elastic password
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
+    user: elastic
+    password: "{{ elasticstack_password.stdout }}"
+    validate_certs: false
+  register: elasticsearch_cluster_status
+  changed_when: false
+  failed_when: false
+  no_log: "{{ elasticstack_no_log }}"
+  when:
+    - elasticsearch_passwords_file.stat.exists | bool
+    - groups['elasticsearch'] | length > 1
+  until: elasticsearch_cluster_status.json.status == "green"
+  retries: 20
+  delay: 10
+
+# We need this check twice. One with failed_when: false to wait for the API to be actually available. And a second time to
+# check the actual return code. Should not cause a huge delay.
+
 - name: Check for cluster status with elastic password
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"

From 6d1c7a620b1c9900fea19573b57a8fcbfaa2fcdf Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 29 Sep 2023 14:42:27 +0200
Subject: [PATCH 04/26] Restrict Elasticsearch restarts

Only restart when we really need it. Especially not after a fresh start.
---
 roles/elasticsearch/defaults/main.yml         |  3 +++
 roles/elasticsearch/handlers/main.yml         |  4 +++-
 .../tasks/elasticsearch-security.yml          | 22 +++++++++----------
 roles/elasticsearch/tasks/main.yml            |  1 +
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml
index bcadfb3d..cb91ad1c 100644
--- a/roles/elasticsearch/defaults/main.yml
+++ b/roles/elasticsearch/defaults/main.yml
@@ -48,6 +48,9 @@ elasticsearch_cert_expiration_buffer: 30
 elasticstack_ca_will_expire_soon: false
 elasticsearch_cert_will_expire_soon: false
 
+# only used internally
+elasticsearch_freshstart: false
+
 # "global" variables for all roles
 
 elasticstack_release: 8
diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml
index b39f884a..a1d326db 100644
--- a/roles/elasticsearch/handlers/main.yml
+++ b/roles/elasticsearch/handlers/main.yml
@@ -5,7 +5,9 @@
     name: elasticsearch
     state: restarted
     daemon_reload: yes
-  when: elasticsearch_enable | bool
+  when:
+    - elasticsearch_enable | bool
+    - not elasticsearch_freshstart | bool
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml
diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index cc6e363c..805df4aa 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -388,24 +388,23 @@
     elasticsearch_http_protocol: "https"
   when: elasticsearch_http_security
 
-- name: Check for cluster status with bootstrap password - no fail
+- name: Check for API with bootstrap password - no fail
   ansible.builtin.uri:
-    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}"
     user: elastic
     password: "{{ elasticsearch_bootstrap_pw }}"
     validate_certs: false
-  register: elasticsearch_cluster_status_bootstrap
+  register: elasticsearch_api_status_bootstrap
   changed_when: false
-  failed_when: false
   no_log: "{{ elasticstack_no_log }}"
   when:
     - not elasticsearch_passwords_file.stat.exists | bool
     - groups['elasticsearch'] | length > 1
-  until: elasticsearch_cluster_status_bootstrap.json.status == "green"
+  until: elasticsearch_api_status_bootstrap.json is defined
   retries: 5
   delay: 10
 
-# We need this check twice. One with failed_when: false to wait for the API to be actually available. And a second time to
+# We need this check twice. One to wait for the API to be actually available. And a second time to
 # check the actual return code. Should not cause a huge delay.
 
 - name: Check for cluster status with bootstrap password
@@ -435,24 +434,23 @@
   delegate_to: "{{ elasticstack_ca }}"
   when: elasticsearch_passwords_file.stat.exists | bool
 
-- name: Check for cluster status with elastic password
+- name: Check for API availability with elastic password
   ansible.builtin.uri:
-    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}"
     user: elastic
     password: "{{ elasticstack_password.stdout }}"
     validate_certs: false
-  register: elasticsearch_cluster_status
+  register: elasticsearch_api_status
   changed_when: false
-  failed_when: false
   no_log: "{{ elasticstack_no_log }}"
   when:
     - elasticsearch_passwords_file.stat.exists | bool
     - groups['elasticsearch'] | length > 1
-  until: elasticsearch_cluster_status.json.status == "green"
+  until: elasticsearch_api_status_bootstrap.json is defined
   retries: 20
   delay: 10
 
-# We need this check twice. One with failed_when: false to wait for the API to be actually available. And a second time to
+# We need this check twice. One to wait for the API to be actually available. And a second time to
 # check the actual return code. Should not cause a huge delay.
 
 - name: Check for cluster status with elastic password
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index c6d9bd59..8a149fef 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -200,6 +200,7 @@
     name: elasticsearch
     state: started
     enabled: yes
+  register: elasticsearch_freshstart
 
 - name: Handle cluster setup without security
   when: not elasticsearch_security | bool

From 0bd410b1668722f9a1c95fe869ff52400c916025 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 29 Sep 2023 14:49:10 +0200
Subject: [PATCH 05/26] Restart Logstash only when needed

---
 roles/logstash/defaults/main.yml | 5 +++++
 roles/logstash/handlers/main.yml | 4 +++-
 roles/logstash/tasks/main.yml    | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml
index 8ff97804..1941e792 100644
--- a/roles/logstash/defaults/main.yml
+++ b/roles/logstash/defaults/main.yml
@@ -73,6 +73,11 @@ logstash_pipeline_identifier: true
 logstash_pipeline_identifier_field_name: "[netways][pipeline]"
 logstash_pipeline_identifier_defaults: false
 
+# Only for internal use
+
+logstash_freshstart:
+  changed: false
+
 elasticstack_ca_dir: /opt/es-ca
 elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords
 elasticstack_ca_pass: PleaseChangeMe
diff --git a/roles/logstash/handlers/main.yml b/roles/logstash/handlers/main.yml
index eb55a868..08b3b71b 100644
--- a/roles/logstash/handlers/main.yml
+++ b/roles/logstash/handlers/main.yml
@@ -4,7 +4,9 @@
   ansible.builtin.service:
     name: logstash
     state: restarted
-  when: logstash_enable | bool
+  when:
+    - logstash_enable | bool
+    - not logstash_freshstart.changed | bool
 
 - name: Restart Logstash noauto
   ansible.builtin.service:
diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml
index 1dcee30b..2695a960 100644
--- a/roles/logstash/tasks/main.yml
+++ b/roles/logstash/tasks/main.yml
@@ -230,3 +230,4 @@
     state: started
     enabled: yes
   when: logstash_enable | bool
+  register: logstash_freshstart

From 2da2cb93b1d20d7cfb7522d84e60e43b81e674ff Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 29 Sep 2023 14:51:33 +0200
Subject: [PATCH 06/26] Fix variable for Elasticsearch fresh start

---
 roles/elasticsearch/defaults/main.yml | 3 ++-
 roles/elasticsearch/handlers/main.yml | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml
index cb91ad1c..ce74f378 100644
--- a/roles/elasticsearch/defaults/main.yml
+++ b/roles/elasticsearch/defaults/main.yml
@@ -49,7 +49,8 @@ elasticstack_ca_will_expire_soon: false
 elasticsearch_cert_will_expire_soon: false
 
 # only used internally
-elasticsearch_freshstart: false
+elasticsearch_freshstart:
+  changed: false
 
 # "global" variables for all roles
 
diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml
index a1d326db..d16168ea 100644
--- a/roles/elasticsearch/handlers/main.yml
+++ b/roles/elasticsearch/handlers/main.yml
@@ -7,7 +7,7 @@
     daemon_reload: yes
   when:
     - elasticsearch_enable | bool
-    - not elasticsearch_freshstart | bool
+    - not elasticsearch_freshstart.changed | bool
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml

From 2140b71055fd40680c7de24aed9bf1c8a607747e Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 29 Sep 2023 18:23:57 +0200
Subject: [PATCH 07/26] Fix variable names

---
 roles/elasticsearch/tasks/elasticsearch-security.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 805df4aa..2cee39d4 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -400,7 +400,7 @@
   when:
     - not elasticsearch_passwords_file.stat.exists | bool
     - groups['elasticsearch'] | length > 1
-  until: elasticsearch_api_status_bootstrap.json is defined
+  until: elasticsearch_api_status_bootstrap.json.cluster_name is defined
   retries: 5
   delay: 10
 
@@ -446,7 +446,7 @@
   when:
     - elasticsearch_passwords_file.stat.exists | bool
     - groups['elasticsearch'] | length > 1
-  until: elasticsearch_api_status_bootstrap.json is defined
+  until: elasticsearch_api_status.json.cluster_name is defined
   retries: 20
   delay: 10
 

From e75994b39c3ae509ffbae216140220efa9afe53b Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 29 Sep 2023 18:26:45 +0200
Subject: [PATCH 08/26] Restart Kibana only when needed

---
 roles/kibana/defaults/main.yml | 3 +++
 roles/kibana/handlers/main.yml | 2 ++
 roles/kibana/tasks/main.yml    | 1 +
 3 files changed, 6 insertions(+)

diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml
index cc21f125..7aa06f7b 100644
--- a/roles/kibana/defaults/main.yml
+++ b/roles/kibana/defaults/main.yml
@@ -18,6 +18,9 @@ kibana_cert_will_expire_soon: false
 kibana_sniff_on_start: false
 kibana_sniff_on_connection_fault: false
 
+kibana_freshstart:
+  changed: false
+
 # "global" variables for all roles
 elasticstack_release: 8
 elasticstack_full_stack: true
diff --git a/roles/kibana/handlers/main.yml b/roles/kibana/handlers/main.yml
index 81ffa146..532d014a 100644
--- a/roles/kibana/handlers/main.yml
+++ b/roles/kibana/handlers/main.yml
@@ -4,3 +4,5 @@
   ansible.builtin.service:
     name: kibana
     state: restarted
+  when:
+    - not kibana_freshstart.changed | bool
diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml
index b5b3ebb9..fff89180 100644
--- a/roles/kibana/tasks/main.yml
+++ b/roles/kibana/tasks/main.yml
@@ -82,6 +82,7 @@
     state: started
     enabled: yes
   when: kibana_enable | bool
+  register: kibana_freshstart
 
 # the following is useful when running tests or extra tasks that need to
 # have Kibana running. Escape it on Rocky8, because it gets time out with Elastic 8

From c2a3976a37080104a911c657ec7050b8b3744c4a Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Sat, 30 Sep 2023 20:52:04 +0200
Subject: [PATCH 09/26] Only restart when all start tasks haven't changed

---
 roles/elasticsearch/defaults/main.yml                | 2 ++
 roles/elasticsearch/handlers/main.yml                | 1 +
 roles/elasticsearch/tasks/elasticsearch-security.yml | 3 ++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml
index ce74f378..3f531606 100644
--- a/roles/elasticsearch/defaults/main.yml
+++ b/roles/elasticsearch/defaults/main.yml
@@ -51,6 +51,8 @@ elasticsearch_cert_will_expire_soon: false
 # only used internally
 elasticsearch_freshstart:
   changed: false
+elasticsearch_freshstart_security:
+  changed: false
 
 # "global" variables for all roles
 
diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml
index d16168ea..ff3b5ab5 100644
--- a/roles/elasticsearch/handlers/main.yml
+++ b/roles/elasticsearch/handlers/main.yml
@@ -8,6 +8,7 @@
   when:
     - elasticsearch_enable | bool
     - not elasticsearch_freshstart.changed | bool
+    - not elasticsearch_freshstart_security.changed | bool
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml
diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 2cee39d4..6eeba0fb 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -352,7 +352,7 @@
     name: elasticsearch
     state: started
     enabled: yes
-  register: elasticsearch_freshstart
+  register: elasticsearch_freshstart_security
 
 - name: Wait for all instances to start
   ansible.builtin.include_tasks: wait_for_instance.yml
@@ -361,6 +361,7 @@
 - name: Restart if Elasticsearch was already running
   when:
     - not elasticsearch_freshstart.changed | bool
+    - not elasticsearch_freshstart_security.changed | bool
   block:
     - name: Force all notified handlers to run at this point, not waiting for normal sync points
       ansible.builtin.meta: flush_handlers

From 6f3115c479f5036674db1d0bc4f9c838c95d06b7 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 2 Oct 2023 14:38:55 +0200
Subject: [PATCH 10/26] Give more ressources to test containers

---
 .github/workflows/test_full_stack.yml      | 2 +-
 .github/workflows/test_roles_pr.yml        | 2 +-
 molecule/elasticstack_default/converge.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml
index 6e78a086..0f30739b 100644
--- a/.github/workflows/test_full_stack.yml
+++ b/.github/workflows/test_full_stack.yml
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 2
+      max-parallel: 1
       matrix:
         distro:
           - rockylinux8
diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml
index ad50534f..156dd994 100644
--- a/.github/workflows/test_roles_pr.yml
+++ b/.github/workflows/test_roles_pr.yml
@@ -48,7 +48,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 2
+      max-parallel: 1
       matrix:
         distro:
           - rockylinux8
diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml
index 61ec7344..73b07ffc 100644
--- a/molecule/elasticstack_default/converge.yml
+++ b/molecule/elasticstack_default/converge.yml
@@ -13,7 +13,7 @@
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
-    elasticsearch_heap: "1"
+    elasticsearch_heap: "2"
     elasticstack_full_stack: true
     elasticstack_no_log: false
     logstash_pipeline_unsafe_shutdown: true

From 28a1ddbc42d035c68e35a253bcf373289783d3d2 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 2 Oct 2023 14:39:37 +0200
Subject: [PATCH 11/26] Typo

---
 roles/elasticsearch/tasks/elasticsearch-security.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 6eeba0fb..e19e094c 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -389,7 +389,7 @@
     elasticsearch_http_protocol: "https"
   when: elasticsearch_http_security
 
-- name: Check for API with bootstrap password - no fail
+- name: Check for API with bootstrap password
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}"
     user: elastic

From 02f96e7b484f57bc948703bf5907079c42267d68 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 2 Oct 2023 15:30:21 +0200
Subject: [PATCH 12/26] Clear cache at end of each role

---
 roles/beats/tasks/main.yml         | 8 ++++++++
 roles/elasticsearch/tasks/main.yml | 8 ++++++++
 roles/kibana/tasks/main.yml        | 8 ++++++++
 roles/logstash/tasks/main.yml      | 8 ++++++++
 4 files changed, 32 insertions(+)

diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml
index f242c0a8..42009808 100644
--- a/roles/beats/tasks/main.yml
+++ b/roles/beats/tasks/main.yml
@@ -82,3 +82,11 @@
 - name: Import Metricbeat tasks
   ansible.builtin.import_tasks: metricbeat.yml
   when: beats_metricbeat | bool
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache # noqa: risky-shell-pipe
+  ansible.builtin.shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 8a149fef..65270122 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -240,6 +240,14 @@
     backup: "{{ elasticsearch_config_backup }}"
   when: elasticsearch_manage_yaml | bool
 
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache # noqa: risky-shell-pipe
+  ansible.builtin.shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
+
 - name: Show Info about heap
   ansible.builtin.debug:
     msg: "Using {{ elasticsearch_heap | int * 1024 }} of {{ ansible_memtotal_mb }} MB as heap for Elasticsearch"
diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml
index fff89180..4127454d 100644
--- a/roles/kibana/tasks/main.yml
+++ b/roles/kibana/tasks/main.yml
@@ -91,3 +91,11 @@
   ansible.builtin.wait_for:
     host: localhost
     port: 5601
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache # noqa: risky-shell-pipe
+  ansible.builtin.shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml
index 2695a960..6120c083 100644
--- a/roles/logstash/tasks/main.yml
+++ b/roles/logstash/tasks/main.yml
@@ -231,3 +231,11 @@
     enabled: yes
   when: logstash_enable | bool
   register: logstash_freshstart
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache # noqa: risky-shell-pipe
+  ansible.builtin.shell: >
+    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"

From 87a4a18219437880334dc6911df2292a3bd9c9c5 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 2 Oct 2023 16:29:17 +0200
Subject: [PATCH 13/26] Workarounds for low ressources on GitHub runners

---
 molecule/elasticstack_default/converge.yml |  1 +
 roles/elasticsearch/tasks/main.yml         | 35 ++++++++++++++++++----
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml
index 73b07ffc..009b0fdd 100644
--- a/molecule/elasticstack_default/converge.yml
+++ b/molecule/elasticstack_default/converge.yml
@@ -12,6 +12,7 @@
   vars:
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elasticsearch_monitoring_enabled: false
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
     elasticsearch_heap: "2"
     elasticstack_full_stack: true
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 65270122..4ba80e7c 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -240,13 +240,36 @@
     backup: "{{ elasticsearch_config_backup }}"
   when: elasticsearch_manage_yaml | bool
 
-# Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
-    rm -rf /var/cache/*
-  changed_when: false
+- name: Work around low ressources on CI/CD nodes
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
+  block:
+    # Free up some space to let elsticsearch allocate replica in GitHub Action
+    - name: Remove cache # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        rm -rf /var/cache/*
+      changed_when: false
+
+    - name: Set watermarks to very high values
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        curl
+        -k
+        -X PUT
+        "https://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
+        -H 'Content-Type: application/json' -d
+        '
+        {
+          "persistent": {
+            "cluster.routing.allocation.disk.watermark.low": "95%",
+            "cluster.routing.allocation.disk.watermark.high": "97%",
+            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
+            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%",
+          }
+        }
+        '
+      changed_when: false
+      no_log: "{{ elasticstack_no_log }}"
 
 - name: Show Info about heap
   ansible.builtin.debug:

From 1929d9fec0156048b6c3385305d65b63fb841ef9 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 2 Oct 2023 16:39:48 +0200
Subject: [PATCH 14/26] Typo

---
 roles/elasticsearch/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 4ba80e7c..bed8495b 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -256,7 +256,7 @@
         curl
         -k
         -X PUT
-        "https://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
+        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
         -H 'Content-Type: application/json' -d
         '
         {

From 510624ff1eb0b11dd4c9c65b9e4c8d5a239ecaea Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Thu, 5 Oct 2023 15:06:35 +0200
Subject: [PATCH 15/26] Minimizing services for small CI/CD runners

---
 molecule/elasticstack_default/molecule.yml |  2 -
 molecule/elasticstack_default/verify.yml   | 78 ++++++++++++----------
 2 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml
index d658c84d..513db812 100644
--- a/molecule/elasticstack_default/molecule.yml
+++ b/molecule/elasticstack_default/molecule.yml
@@ -10,7 +10,6 @@ platforms:
     groups:
       - beats
       - logstash
-      - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
@@ -22,7 +21,6 @@ platforms:
   - name: "elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}"
     groups:
       - beats
-      - logstash
       - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml
index abceef3f..3a2e8430 100644
--- a/molecule/elasticstack_default/verify.yml
+++ b/molecule/elasticstack_default/verify.yml
@@ -59,42 +59,46 @@
         msg: "Elasticsearch received {{ logstash_count.stdout }} events so far"
       when: "'elasticsearch' in group_names"
 
-    - name: fetch kibana.yml
-      ansible.builtin.command: cat /etc/kibana/kibana.yml
-      register: kibanayml
-
-    - name: Show kibana.yml
-      ansible.builtin.debug:
-        var: kibanayml.stdout_lines
-
-    - name: Check for Kibana port
-      ansible.builtin.wait_for:
-        port: 5601
-        timeout: 120
-
-    - name: Connect to Kibana
-      ansible.builtin.command:
-        curl
-        -s
-        -u elastic:{{ elastic_pass.stdout }}
-        http://{{ ansible_hostname }}:5601/api/status
-      register: curl_out
-      failed_when:
-        - "'green' not in curl_out.stdout"
-        - "'Elasticsearch is available' not in curl_out.stdout"
-
-      # The following might be nicer but doesn't work
-      #- name: Connect to Kibana
-      #  ansible.builtin.uri:
-      #    url: http://ansible-role-kibana_full_stack:5601/api/status
-      #    user: elastic
-      #    password: "{{ elastic_password.stdout }}"
-      #    return_content: yes
-      #  register: kibana_status
-      #  #failed_when: "'"title": "Green"' not in kibana_status.content"
-      #  failed_when: "'Green' not in kibana_status.content"
-
-    - name: Health check
+    - name: Run Kibana checks
+      when: "'kibana' in group_names"
+      block:
+
+        - name: Fetch kibana.yml
+          ansible.builtin.command: cat /etc/kibana/kibana.yml
+          register: kibanayml
+
+        - name: Show kibana.yml
+          ansible.builtin.debug:
+            var: kibanayml.stdout_lines
+
+        - name: Check for Kibana port
+          ansible.builtin.wait_for:
+            port: 5601
+            timeout: 120
+
+        - name: Connect to Kibana
+          ansible.builtin.command:
+            curl
+            -s
+            -u elastic:{{ elastic_pass.stdout }}
+            http://{{ ansible_hostname }}:5601/api/status
+          register: curl_out
+          failed_when:
+            - "'green' not in curl_out.stdout"
+            - "'Elasticsearch is available' not in curl_out.stdout"
+
+          # The following might be nicer but doesn't work
+          #- name: Connect to Kibana
+          #  ansible.builtin.uri:
+          #    url: http://ansible-role-kibana_full_stack:5601/api/status
+          #    user: elastic
+          #    password: "{{ elastic_password.stdout }}"
+          #    return_content: yes
+          #  register: kibana_status
+          #  #failed_when: "'"title": "Green"' not in kibana_status.content"
+          #  failed_when: "'Green' not in kibana_status.content"
+
+    - name: Elasticsearch health check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health
         method: GET
@@ -110,7 +114,7 @@
       delay: 10
       when: groups['elasticsearch'] | length > 1
 
-    - name: Node check
+    - name: Elasticsearch Node check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes
         method: GET

From d0823f63538e7f168b7b5f22cfc4d7d8acfa0162 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 10 Oct 2023 13:16:46 +0200
Subject: [PATCH 16/26] Add debug tasks

---
 .../tasks/elasticsearch-security.yml           | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index e19e094c..6ac95973 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -451,6 +451,24 @@
   retries: 20
   delay: 10
 
+### DEBUGGING ###
+- name: Check for cluster explain
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/allocation/explain?pretty"
+    user: elastic
+    password: "{{ elasticstack_password.stdout }}"
+    validate_certs: false
+  register: elasticsearch_cluster_explain
+  changed_when: false
+  no_log: "{{ elasticstack_no_log }}"
+  when:
+    - elasticsearch_passwords_file.stat.exists | bool
+    - groups['elasticsearch'] | length > 1
+
+- name: Output debug information
+  ansible.builtin.debug:
+    var: elasticsearch_cluster_explain
+
 # We need this check twice. One to wait for the API to be actually available. And a second time to
 # check the actual return code. Should not cause a huge delay.
 

From a75e06eef693af7d2b06ee041d485431c7cd27a7 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 10 Oct 2023 13:30:23 +0200
Subject: [PATCH 17/26] Activate logging in another scenario

---
 molecule/elasticsearch_default/converge.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml
index 8cba6694..1e836b59 100644
--- a/molecule/elasticsearch_default/converge.yml
+++ b/molecule/elasticsearch_default/converge.yml
@@ -12,6 +12,7 @@
     elasticsearch_disable_systemcallfilterchecks: true
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
     elasticsearch_heap: "1"
+    elasticstack_no_log: false
   tasks:
     - name: Include Elastics repos role
       ansible.builtin.include_role:

From 74671b2f10e8c9646e2e22e9a8ece1e41112dc2d Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 10 Oct 2023 14:35:58 +0200
Subject: [PATCH 18/26] Don't fail when there's nothing to explain

---
 roles/elasticsearch/tasks/elasticsearch-security.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 6ac95973..0aa0913e 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -460,6 +460,7 @@
     validate_certs: false
   register: elasticsearch_cluster_explain
   changed_when: false
+  failed_when: false
   no_log: "{{ elasticstack_no_log }}"
   when:
     - elasticsearch_passwords_file.stat.exists | bool

From dd8b1c58a5fc70e112c9a06d012f591c524c9504 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 10 Oct 2023 17:04:53 +0200
Subject: [PATCH 19/26] Set watermarks persistent and transient

---
 roles/elasticsearch/tasks/main.yml | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index bed8495b..42d20dbd 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -250,7 +250,7 @@
         rm -rf /var/cache/*
       changed_when: false
 
-    - name: Set watermarks to very high values
+    - name: Set persistent watermarks to very high values in Docker
       ansible.builtin.shell: >
         if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
         curl
@@ -271,6 +271,27 @@
       changed_when: false
       no_log: "{{ elasticstack_no_log }}"
 
+    - name: Set transient watermarks to very high values in Docker
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        curl
+        -k
+        -X PUT
+        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
+        -H 'Content-Type: application/json' -d
+        '
+        {
+          "transient": {
+            "cluster.routing.allocation.disk.watermark.low": "95%",
+            "cluster.routing.allocation.disk.watermark.high": "97%",
+            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
+            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%",
+          }
+        }
+        '
+      changed_when: false
+      no_log: "{{ elasticstack_no_log }}"
+
 - name: Show Info about heap
   ansible.builtin.debug:
     msg: "Using {{ elasticsearch_heap | int * 1024 }} of {{ ansible_memtotal_mb }} MB as heap for Elasticsearch"

From 3ab78f57d2f98d2bac0bede112ff028bc4d214e0 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 13:38:14 +0200
Subject: [PATCH 20/26] Fix curl and add debug

---
 roles/elasticsearch/tasks/main.yml | 36 +++++++++---------------------
 1 file changed, 11 insertions(+), 25 deletions(-)

diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 42d20dbd..5fbfbb4e 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -241,7 +241,7 @@
   when: elasticsearch_manage_yaml | bool
 
 - name: Work around low ressources on CI/CD nodes
-  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
+  #when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
   block:
     # Free up some space to let elsticsearch allocate replica in GitHub Action
     - name: Remove cache # noqa: risky-shell-pipe
@@ -256,41 +256,27 @@
         curl
         -k
         -X PUT
-        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
+        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings"
         -H 'Content-Type: application/json' -d
         '
         {
           "persistent": {
-            "cluster.routing.allocation.disk.watermark.low": "95%",
-            "cluster.routing.allocation.disk.watermark.high": "97%",
+            "cluster.routing.allocation.disk.watermark.low": "97%",
+            "cluster.routing.allocation.disk.watermark.high": "98%",
             "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
-            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%",
+            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%"
           }
         }
         '
       changed_when: false
+      register: debugme
       no_log: "{{ elasticstack_no_log }}"
+      when:
+        - elasticstack_password.stdout is defined
 
-    - name: Set transient watermarks to very high values in Docker
-      ansible.builtin.shell: >
-        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
-        curl
-        -k
-        -X PUT
-        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticsearch_bootstrap_pw }}@localhost:9200/_cluster/settings"
-        -H 'Content-Type: application/json' -d
-        '
-        {
-          "transient": {
-            "cluster.routing.allocation.disk.watermark.low": "95%",
-            "cluster.routing.allocation.disk.watermark.high": "97%",
-            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
-            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%",
-          }
-        }
-        '
-      changed_when: false
-      no_log: "{{ elasticstack_no_log }}"
+    - name: Show debug information
+      debug:
+        var: debugme
 
 - name: Show Info about heap
   ansible.builtin.debug:

From 7ff8527e5541d09bd491018af59f360f433d13cf Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 13:44:34 +0200
Subject: [PATCH 21/26] Lint

---
 roles/elasticsearch/tasks/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 5fbfbb4e..b4fc4de6 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -250,7 +250,7 @@
         rm -rf /var/cache/*
       changed_when: false
 
-    - name: Set persistent watermarks to very high values in Docker
+    - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe
       ansible.builtin.shell: >
         if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
         curl
@@ -275,7 +275,7 @@
         - elasticstack_password.stdout is defined
 
     - name: Show debug information
-      debug:
+      ansible.builtin.debug:
         var: debugme
 
 - name: Show Info about heap

From 18753c7600a09fbdcde3d79ff4e9dc830941834a Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 14:55:48 +0200
Subject: [PATCH 22/26] Move watermark change

---
 .../tasks/elasticsearch-security.yml          | 38 +++++++++++++++++++
 roles/elasticsearch/tasks/main.yml            | 38 -------------------
 2 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 0aa0913e..19aca3b7 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -451,6 +451,44 @@
   retries: 20
   delay: 10
 
+- name: Work around low ressources on CI/CD nodes
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
+  block:
+    # Free up some space to let elsticsearch allocate replica in GitHub Action
+    - name: Remove cache # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        rm -rf /var/cache/*
+      changed_when: false
+
+    - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        curl
+        -k
+        -X PUT
+        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings"
+        -H 'Content-Type: application/json' -d
+        '
+        {
+          "persistent": {
+            "cluster.routing.allocation.disk.watermark.low": "97%",
+            "cluster.routing.allocation.disk.watermark.high": "98%",
+            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
+            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%"
+          }
+        }
+        '
+      changed_when: false
+      register: debugme
+      no_log: "{{ elasticstack_no_log }}"
+      when:
+        - elasticstack_password.stdout is defined
+
+    - name: Show debug information
+      ansible.builtin.debug:
+        var: debugme
+
 ### DEBUGGING ###
 - name: Check for cluster explain
   ansible.builtin.uri:
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index b4fc4de6..8a149fef 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -240,44 +240,6 @@
     backup: "{{ elasticsearch_config_backup }}"
   when: elasticsearch_manage_yaml | bool
 
-- name: Work around low ressources on CI/CD nodes
-  #when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
-  block:
-    # Free up some space to let elsticsearch allocate replica in GitHub Action
-    - name: Remove cache # noqa: risky-shell-pipe
-      ansible.builtin.shell: >
-        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
-        rm -rf /var/cache/*
-      changed_when: false
-
-    - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe
-      ansible.builtin.shell: >
-        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
-        curl
-        -k
-        -X PUT
-        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings"
-        -H 'Content-Type: application/json' -d
-        '
-        {
-          "persistent": {
-            "cluster.routing.allocation.disk.watermark.low": "97%",
-            "cluster.routing.allocation.disk.watermark.high": "98%",
-            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
-            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%"
-          }
-        }
-        '
-      changed_when: false
-      register: debugme
-      no_log: "{{ elasticstack_no_log }}"
-      when:
-        - elasticstack_password.stdout is defined
-
-    - name: Show debug information
-      ansible.builtin.debug:
-        var: debugme
-
 - name: Show Info about heap
   ansible.builtin.debug:
     msg: "Using {{ elasticsearch_heap | int * 1024 }} of {{ ansible_memtotal_mb }} MB as heap for Elasticsearch"

From 9831cd97800123a7c285f370c402e6616eb82c1d Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 16:25:25 +0200
Subject: [PATCH 23/26] Remove debug and throttleing

---
 .github/workflows/test_full_stack.yml         |  2 +-
 .github/workflows/test_roles_pr.yml           |  2 +-
 .../tasks/elasticsearch-security.yml          | 24 -------------------
 3 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml
index 0f30739b..771b6f81 100644
--- a/.github/workflows/test_full_stack.yml
+++ b/.github/workflows/test_full_stack.yml
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
+        #max-parallel: 1
       matrix:
         distro:
           - rockylinux8
diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml
index 156dd994..1aa529d9 100644
--- a/.github/workflows/test_roles_pr.yml
+++ b/.github/workflows/test_roles_pr.yml
@@ -48,7 +48,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
+        #max-parallel: 1
       matrix:
         distro:
           - rockylinux8
diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 19aca3b7..e8e83c16 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -480,34 +480,10 @@
         }
         '
       changed_when: false
-      register: debugme
       no_log: "{{ elasticstack_no_log }}"
       when:
         - elasticstack_password.stdout is defined
 
-    - name: Show debug information
-      ansible.builtin.debug:
-        var: debugme
-
-### DEBUGGING ###
-- name: Check for cluster explain
-  ansible.builtin.uri:
-    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/allocation/explain?pretty"
-    user: elastic
-    password: "{{ elasticstack_password.stdout }}"
-    validate_certs: false
-  register: elasticsearch_cluster_explain
-  changed_when: false
-  failed_when: false
-  no_log: "{{ elasticstack_no_log }}"
-  when:
-    - elasticsearch_passwords_file.stat.exists | bool
-    - groups['elasticsearch'] | length > 1
-
-- name: Output debug information
-  ansible.builtin.debug:
-    var: elasticsearch_cluster_explain
-
 # We need this check twice. One to wait for the API to be actually available. And a second time to
 # check the actual return code. Should not cause a huge delay.
 

From 82589295af443948a75b7eeec8cf4f155f7375a5 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 17:06:52 +0200
Subject: [PATCH 24/26] Undo changes to throtteling.

---
 .github/workflows/test_full_stack.yml | 2 +-
 .github/workflows/test_roles_pr.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml
index 771b6f81..311b3233 100644
--- a/.github/workflows/test_full_stack.yml
+++ b/.github/workflows/test_full_stack.yml
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       fail-fast: false
-        #max-parallel: 1
+        max-parallel: 2
       matrix:
         distro:
           - rockylinux8
diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml
index 1aa529d9..aa3e8483 100644
--- a/.github/workflows/test_roles_pr.yml
+++ b/.github/workflows/test_roles_pr.yml
@@ -48,7 +48,7 @@ jobs:
 
     strategy:
       fail-fast: false
-        #max-parallel: 1
+        max-parallel: 2
       matrix:
         distro:
           - rockylinux8

From cf54655ce5f8c901b305d9e7d0ad18e2193a04d6 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 11 Oct 2023 18:12:06 +0200
Subject: [PATCH 25/26] Typo

---
 .github/workflows/test_full_stack.yml | 2 +-
 .github/workflows/test_roles_pr.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml
index 311b3233..6e78a086 100644
--- a/.github/workflows/test_full_stack.yml
+++ b/.github/workflows/test_full_stack.yml
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       fail-fast: false
-        max-parallel: 2
+      max-parallel: 2
       matrix:
         distro:
           - rockylinux8
diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml
index aa3e8483..ad50534f 100644
--- a/.github/workflows/test_roles_pr.yml
+++ b/.github/workflows/test_roles_pr.yml
@@ -48,7 +48,7 @@ jobs:
 
     strategy:
       fail-fast: false
-        max-parallel: 2
+      max-parallel: 2
       matrix:
         distro:
           - rockylinux8

From 51967e27336092901458e5d93b0659e5a772e1fd Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Fri, 13 Oct 2023 13:25:37 +0200
Subject: [PATCH 26/26] Replace unnecessary shell with command

---
 roles/beats/tasks/main.yml                           | 5 ++---
 roles/elasticsearch/tasks/elasticsearch-security.yml | 5 ++---
 roles/elasticsearch/tasks/main.yml                   | 5 ++---
 roles/kibana/tasks/main.yml                          | 5 ++---
 roles/logstash/tasks/main.yml                        | 5 ++---
 5 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml
index 42009808..9d521bb1 100644
--- a/roles/beats/tasks/main.yml
+++ b/roles/beats/tasks/main.yml
@@ -84,9 +84,8 @@
   when: beats_metricbeat | bool
 
 # Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+- name: Remove cache
+  ansible.builtin.command: >
     rm -rf /var/cache/*
   changed_when: false
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index e8e83c16..48bcb2aa 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -455,9 +455,8 @@
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
   block:
     # Free up some space to let elsticsearch allocate replica in GitHub Action
-    - name: Remove cache # noqa: risky-shell-pipe
-      ansible.builtin.shell: >
-        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+    - name: Remove cache
+      ansible.builtin.command: >
         rm -rf /var/cache/*
       changed_when: false
 
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 8a149fef..0632f870 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -178,9 +178,8 @@
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
 
 # Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+- name: Remove cache
+  ansible.builtin.command: >
     rm -rf /var/cache/*
   changed_when: false
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml
index 4127454d..74e91344 100644
--- a/roles/kibana/tasks/main.yml
+++ b/roles/kibana/tasks/main.yml
@@ -93,9 +93,8 @@
     port: 5601
 
 # Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+- name: Remove cache
+  ansible.builtin.command: >
     rm -rf /var/cache/*
   changed_when: false
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml
index 6120c083..a1d1b3de 100644
--- a/roles/logstash/tasks/main.yml
+++ b/roles/logstash/tasks/main.yml
@@ -233,9 +233,8 @@
   register: logstash_freshstart
 
 # Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+- name: Remove cache
+  ansible.builtin.command: >
     rm -rf /var/cache/*
   changed_when: false
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"