Merge pull request #5 from NETWAYS/upgradeEL8/changes

Upgrade el8/changes

Author: widhalmt

.github/workflows/test_full_stack.yml

@@ -29,6 +29,9 @@ jobs:
           - rockylinux8
         scenario:
           - elasticstack_default
+        release:
+          - 7
+          - 8
 
     steps:
       - name: Check out code
@@ -56,3 +59,4 @@ jobs:
           MOLECULE_DISTRO: ${{ matrix.distro }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          ELASTIC_RELEASE: ${{ matrix.release }}

.github/workflows/test_role_beats.yml

@@ -71,6 +71,7 @@ jobs:
       matrix:
         distro: [debian11, rockylinux8, ubuntu2204]
         scenario: [beats_default, beats_full_stack, beats_peculiar]
+        release: [ 7, 8 ]
 
     steps:
       - name: Check out code
@@ -98,3 +99,4 @@ jobs:
           MOLECULE_DISTRO: ${{ matrix.distro }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          ELASTIC_RELEASE: ${{ matrix.release }}

.github/workflows/test_role_elasticsearch.yml

@@ -71,6 +71,7 @@ jobs:
       matrix:
         distro: [ubuntu2204]
         scenario: [elasticsearch_default, elasticsearch_cluster, elasticsearch_cluster-oss, elasticsearch_no-security]
+        release: [7, 8]
 
     steps:
       - name: Check out code
@@ -96,5 +97,6 @@ jobs:
           molecule test -s ${{ matrix.scenario }}
         env:
           MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'

.github/workflows/test_role_kibana.yml

@@ -71,6 +71,7 @@ jobs:
       matrix:
         distro: [ubuntu2204]
         scenario: [kibana_default, kibana_full_stack, kibana_full_stack-oss]
+        release: [ 7, 8 ]
 
     steps:
       - name: Check out code
@@ -96,3 +97,4 @@ jobs:
           molecule test -s ${{ matrix.scenario }}
         env:
           MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}

.github/workflows/test_role_logstash.yml

@@ -69,6 +69,7 @@ jobs:
       matrix:
         distro: [ubuntu2204]
         scenario: [logstash_default]
+        release: [ 7, 8 ]
 
     steps:
       - name: Check out code
@@ -96,6 +97,7 @@ jobs:
           MOLECULE_DISTRO: ${{ matrix.distro }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          ELASTIC_RELEASE: ${{ matrix.release }}
 
   molecule_tests:
     runs-on: ubuntu-latest
@@ -115,6 +117,7 @@ jobs:
                    logstash_full_stack-oss,
                    logstash_specific_version,
                    logstash_pipelines]
+        release: [ 7, 8 ]
 
     steps:
       - name: Check out code
@@ -142,3 +145,4 @@ jobs:
           MOLECULE_DISTRO: ${{ matrix.distro }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          ELASTIC_RELEASE: ${{ matrix.release }}

.github/workflows/test_role_repos.yml

@@ -70,7 +70,8 @@ jobs:
       matrix:
 
         distro: [centos7, debian10, debian11, rockylinux8, rockylinux9, ubuntu2004, ubuntu2204]
-        scenario: [repos_default, repos_oss, repos_elastic8]
+        scenario: [repos_default, repos_oss]
+        release: [ 7, 8 ]
 
     steps:
       - name: Check out code
@@ -97,3 +98,4 @@ jobs:
           molecule test -s ${{ matrix.scenario }}
         env:
           MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}

.github/workflows/test_roles_pr.yml

@@ -28,6 +28,9 @@ jobs:
           - kibana_default
           - logstash_default
           - repos_default
+        release:
+          - 7
+          - 8
 
     steps:
       - name: Check out code
@@ -55,3 +58,4 @@ jobs:
           MOLECULE_DISTRO: ${{ matrix.distro }}
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
+          ELASTIC_RELEASE: ${{ matrix.release }}

README.md

@@ -39,13 +39,32 @@ You may want the following Ansible roles installed. There other ways to achieve
 
 ## Usage
 
+### Default Passwords 
+
+Default Passwords  can be seen during generation, or found later in `/usr/share/elasticsearch/initial_passwords`
+
+To turn off security currently:
+`elastic_override_beats_tls: true`
+### Redis
+
+0) You need to install the redis role which is maintained by geerlingguy.
+
+```
+ansible-galaxy install geerlingguy.redis 
+```
+
+1) Default: For general Elastic Stack installations using all features use the following. You will need Redis installed and running for the default setup to run. A viable way is using the `geerlingguy.redis` role. (You can install it with `ansible-galaxy install geerlingguy.redis)
+
+2) Specific: For OSS Installation without X-Pack features you can use the following. _Note_ this is only available for version `7.x`.
+
 Our default configuration will collect filesystem logs placed by `rsyslog`. Therefor our example playbook makes sure, `rsyslog` is installed. If you don't want that, please change the configuration of the `beats` module. Without syslog you won't receive any messages with the default configuration.
 
 There are some comments in the Playbook. Either fill them with the correct values (`remote_user`) or consider them as a hint to commonly used options.
 
 _Note_: The roles rely on hardcoded group names for placing services on hosts. Please make sure you have groups named `elasticsearch`, `logstash` and `kibana` in your Ansible inventory. Hosts in these groups will get the respective services. Restricting your plays to the appropriate hosts will not work because the roles interact with hosts from other groups e.g. for certificate generation.
 
 The execution order of the roles is important! (see below)
+
 ```
 ---
 - hosts: all
@@ -74,5 +93,4 @@ The execution order of the roles is important! (see below)
     - logstash
     - kibana
     - beats
-
 ```

docs/role-elasticsearch.md

@@ -7,6 +7,8 @@ This role installs manages Elasticsearch on your hosts. Optionally it can config
 
 If you use the role to set up security you can use its CA to create certificates for Logstash and Kibana, too.
 
+Please note that setting `elasticsearch_bootstrap_pw` as variable will only take effect when initialising Elasticsearch. Changes after starting elasticsearch for the first time will not change the bootstrap password for the instance and will lead to breaking tests.
+
 Role Variables
 --------------
 
@@ -16,6 +18,7 @@ Role Variables
 * *elasticsearch_datapath*: Path where Elasticsearch will store it's data. (default: `/var/lib/elasticsearch` - the packages default)
 * *elasticsearch_create_datapath*: Create the path for data to store if it doesn't exist. (default: `false` - only useful if you change `elasticsearch_datapath`)
 * *elasticsearch_fs_repo*: List of paths that should be registered as repository for snapshots (only filesystem supported so far). (default: none) Remember, that every node needs access to the same share under the same path.
+* *elasticsearch_bootstrap_pw*: Bootstrap password for Elasticsearch (Default: `PleaseChangeMe`)
 * *elasticsearch_disable_systemcallfilterchecks*: Disable system call filter checks. This has a security impact but is necessary on some systems. Please refer to the [docs](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html) for details. (default: `false`)
 * *elasticsearch_pamlimits*: Set pam_limits neccessary for Elasticsearch. (Default: `true`)
 

molecule/beats_default/converge.yml

@@ -8,8 +8,10 @@
   collections:
     - NETWAYS.elasticstack
   vars:
+    elastic_stack_full_stack: false
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/beats_full_stack/converge.yml

@@ -17,6 +17,7 @@
       - "testbed: molecule"
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/beats_peculiar/converge.yml

@@ -33,6 +33,7 @@
     beats_metricbeat: true
     metricbeat_output: logstash
     filebeat_docker: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Set Filebeat version on RedHat
       set_fact:

molecule/elasticsearch_cluster-8/converge.yml

@@ -9,6 +9,7 @@
     elastic_variant: oss
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: 7
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/elasticsearch_cluster-8/molecule.yml

@@ -8,6 +8,7 @@
   vars:
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/elasticsearch_cluster-8/prepare.yml

@@ -6,10 +6,11 @@
     - NETWAYS.elasticstack
   hosts: all
   vars:
-    elasticsearch_enable: false
-    elasticsearch_security: false
+    #elasticsearch_security: true # needed for tests of > 7 releases
+    elastic_stack_full_stack: false
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/elasticsearch_cluster-8/requirements.yml

@@ -7,8 +7,10 @@
   hosts: all
   vars:
     elasticsearch_security: false
+    elastic_security: false
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastics repos role
       include_role:

molecule/elasticsearch_cluster-oss/converge.yml

@@ -4,7 +4,17 @@ dependency:
 driver:
   name: docker
 platforms:
-  - name: elasticsearch_no_security
+  - name: elasticsearch-nosecurity1
+    groups:
+      - elasticsearch
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroupns_mode: host
+    privileged: true
+    pre_build_image: true
+  - name: elasticsearch-nosecurity2
     groups:
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"

molecule/elasticsearch_cluster/converge.yml

@@ -7,18 +7,6 @@
     elastic_initial_passwords: /usr/share/elasticsearch/initial_passwords
   tasks:
 
-  - name: Set elasticsearch_ca variable if not already done by user
-    set_fact:
-      elasticsearch_ca: "{{ groups['elasticsearch'][0] }}"
-    when: elasticsearch_ca is undefined
-
-  - name: Fetch Elastic password
-    shell: grep "PASSWORD elastic" {{ elastic_initial_passwords }} | awk {' print $4 '}
-    register: elastic_password
-    changed_when: false
-    delegate_to: "{{ elasticsearch_ca }}"
-
-
 # Remember, this is the no-security scenario. So no https
   - name: Health check
     uri:

molecule/elasticsearch_default/converge.yml

@@ -8,6 +8,7 @@
     elasticsearch_disable_systemcallfilterchecks: true
     elastic_stack_full_stack: true
     elastic_variant: oss
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastic Repos
       include_role:

molecule/elasticsearch_no-security/converge.yml

@@ -5,6 +5,9 @@
 # https://github.com/ansible-community/molecule/issues/1567#issuecomment-436876722
 - name: Converge
   hosts: all
+  vars:
+    elastic_stack_full_stack: false
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   collections:
     - NETWAYS.elasticstack
   tasks:

molecule/elasticsearch_no-security/molecule.yml

@@ -10,6 +10,7 @@
   vars:
     elastic_stack_full_stack: true
     elastic_variant: oss
+    elastic_release: 7
   tasks:
     - name: Include Elastic Repos
       include_role:

molecule/elasticsearch_no-security/verify.yml

@@ -9,6 +9,7 @@
     - NETWAYS.elasticstack
   vars:
     elastic_stack_full_stack: true
+    elastic_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
   tasks:
     - name: Include Elastic Repos
       include_role: