Update tasks to use cert_info plugin (#163)

Fixes #161. Update certificate handling tasks and docs

---------

Co-authored-by: Afeef Ghannam <[email protected]>
Co-authored-by: Afeef Ghannam <[email protected]>

Author: danopt

.github/workflows/test_full_stack.yml

@@ -44,9 +44,9 @@ jobs:
       max-parallel: 1
       matrix:
         distro:
+          - rockylinux8
           - ubuntu2004
           - ubuntu2204
-          - rockylinux8
           - debian10
         scenario:
           - elasticstack_default

docs/role-beats.md

@@ -10,6 +10,9 @@ Requirements
 
 You need to have the beats you want to install available in your software repositories. We provide a [role](./role-repos.md) for just that but if you have other ways of managing software, just make sure it's available. Alternatively you can install the Beats yourself.
 
+* `cryptography` >= 2.5 
+* `community.crypto` collection: ansible-galaxy collection install community.crypto
+
 Role Variables
 --------------
 
@@ -101,7 +104,7 @@ If you want to use this role with your own TLS certificates, use these variables
 * *beats_tls_key*: Path to the keyfiles (default: `{{ beats_ca_dir }}/{{ ansible_hostname }}.key`)
 * *beats_tls_cert*: Path to the certificate (default: `{{ beats_ca_dir }}/{{ ansible_hostname }}.crt`)
 * *beats_tls_key_passphrase*: Passphrase of the keyfile (default: `BeatsChangeMe`)
-* *beats_cert_expiration_buffer*: Ansible will renew the beats certificate if its validity is shorter than this value, which should be number of days. (default: 30)
+* *beats_cert_expiration_buffer*: Ansible will renew the beats certificate if its validity is shorter than this value (default: `+30d`). The valid format is `+[w | d | h | m | s]`, example `+20w5d7h`. 
 * *beats_cert_will_expire_soon*: Set it to true to renew beats certificate (default: `false`), Or run the playbook with `--tags renew_beats_cert` to do that.
 * *beats_tls_cacert*: Path to the CA.crt (default: `{{ beats_ca_dir }}/ca.crt`)
 

docs/role-elasticsearch.md

@@ -9,13 +9,18 @@ If you use the role to set up security you, can use its CA to create certificate
 
 Please note that setting `elasticsearch_bootstrap_pw` as variable will only take effect when initialising Elasticsearch. Changes after starting elasticsearch for the first time will not change the bootstrap password for the instance and will lead to breaking tests.
 
+Requirements
+------------
+
+* `cryptography` >= 2.5
+
 Role Variables
 --------------
 
 * *elasticsearch_enable*: Start and enable Elasticsearch (default: `true`)
 * *elasticsearch_heap*: Heapsize for Elasticsearch. (Half of free memory on host. Maximum 30GB. (default: Half of hosts memory. Min 1GB, Max 30GB)
 * *elasticsearch_tls_key_passphrase*: Passphrase for elasticsearch certificates (default: `PleaseChangeMeIndividually`)
-* *elasticsearch_cert_expiration_buffer*: Ansible will renew the elasticsearch certificate if its validity is shorter than this value, which should be number of days. (default: 30)
+* *elasticsearch_cert_expiration_buffer*: Ansible will renew the elasticsearch certificate if its validity is shorter than this value, which should be number of days. (default: `30`)
 * *elasticsearch_cert_will_expire_soon*: Set it to true to renew elasticsearch certificate (default: `false`), Or run the playbook with `--tags renew_elasticsearch_cert` to do that.
 * *elasticsearch_datapath*: Path where Elasticsearch will store it's data. (default: `/var/lib/elasticsearch` - the packages default)
 * *elasticsearch_create_datapath*: Create the path for data to store if it doesn't exist. (default: `false` - only useful if you change `elasticsearch_datapath`)

docs/role-kibana.md

@@ -5,6 +5,10 @@ Ansible Role: Kibana
 
 This roles installs and configures Kibana.
 
+Requirements
+------------
+
+* `cryptography` >= 2.5
 
 Role Variables
 --------------
@@ -23,7 +27,7 @@ These variables are identical over all our elastic related roles, hence the diff
 * *elasticstack_full_stack*: Use `ansible-role-elasticsearch` as well (default: `false`)
 * *elasticstack_elasticsearch_http_port*: Port of Elasticsearch http (Default: `9200`)
 * *kibana_tls_key_passphrase*: Passphrase for kibana certificates (default: `PleaseChangeMe`)
-* *kibana_cert_expiration_buffer*: Ansible will renew the kibana certificate if its validity is shorter than this value, which should be number of days. (default: 30)
+* *kibana_cert_expiration_buffer*: Ansible will renew the kibana certificate if its validity is shorter than this value, which should be number of days. (default: `30`)
 * *kibana_cert_will_expire_soon*: Set it to true to renew kibana certificate (default: `false`), Or run the playbook with `--tags renew_kibana_cert` to do that.
 * *elasticstack_kibana_host*: Hostname users use to connect to Kibana (default: FQDN of the host the role is executed on)
 * *elasticstack_kibana_port*: Port Kibana webinterface is listening on (default: `5601`)

docs/role-logstash.md

@@ -19,6 +19,7 @@ Requirements
 ------------
 
 * `community.general` collection
+* `cryptography` >= 2.5
 
 You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md)
 

molecule/elasticstack_default/converge.yml

@@ -48,6 +48,6 @@
       service:
         name: rsyslog
         state: started
-    - name: Include Kibana
+    - name: Include kibana
       include_role:
         name: kibana

molecule/elasticstack_default/prepare.yml

@@ -33,3 +33,19 @@
           - git
           - openssl
           - unzip
+          - systemd
+
+    - name: Update all installed packages RHEL
+      yum:
+        name: '*'
+        state: latest
+        update_cache: yes
+        update_only: yes
+      when: ansible_os_family == "RedHat"
+
+    - name: Update all installed packages Debian
+      apt:
+        name: '*'
+        state: latest
+        update_cache: yes
+      when: ansible_os_family == "Debian"

molecule/elasticstack_default/requirements.yml

@@ -6,3 +6,4 @@ roles:
 
 collections:
   - community.general
+  - community.crypto

plugins/module_utils/certs.py

@@ -169,7 +169,7 @@ def extensions_info(self):
                     # get critical value
                     critical = to_text(extension.critical)
                     self.result['extensions'][name]['_critical'] = to_text(critical)
-                self.extensions_values_info(name, extension)
+                    self.extensions_values_info(name, extension)
         except Exception as e:
             # if something went wrong skip this extension and its key values and
             # also create a warning

roles/beats/defaults/main.yml

@@ -68,7 +68,7 @@ elasticstack_ca_dir: /opt/es-ca
 elasticstack_ca_pass: PleaseChangeMe
 elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords
 elasticstack_elasticsearch_http_port: 9200
-beats_cert_expiration_buffer: 30
+beats_cert_expiration_buffer: "+30d"
 beats_cert_will_expire_soon: false
 
 # Variables for debugging and development

roles/beats/tasks/beats-security.yml

@@ -13,35 +13,25 @@
     path: "/etc/beats/certs/{{ inventory_hostname }}-beats.crt"
   register: beats_cert_exists
 
-- name: Get the beats certificate expiration date # noqa: risky-shell-pipe
-  shell: >-
-    if test -v BASH; then set -o pipefail; fi;
-    openssl x509
-    -in '/etc/beats/certs/{{ inventory_hostname }}-beats.crt'
-    -noout -enddate |
-    awk -F'=' '{print $2}'
+- name: Get the beats certificate expiration date
+  community.crypto.x509_certificate_info:
+    path: "/etc/beats/certs/{{ inventory_hostname }}-beats.crt"
+    valid_at:
+      check_period: "{{ beats_cert_expiration_buffer }}"
   register: beats_cert_expiration_date
-  args:
-    executable: /bin/bash
-  changed_when: false
   when: beats_cert_exists.stat.exists | bool
 
-- name: Set the beats certificate expiration date in days
-  set_fact:
-    beats_cert_expiration_days: "{{ ((beats_cert_expiration_date.stdout | to_datetime('%b %d %H:%M:%S %Y %Z')) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
-  when: beats_cert_expiration_date.skipped is not defined
-
 - name: Set beats certificate will expire soon to true
   set_fact:
     beats_cert_will_expire_soon: true
-  when: beats_cert_expiration_days is defined and beats_cert_expiration_days | int <= beats_cert_expiration_buffer | int
+  when: beats_cert_expiration_date.skipped is not defined and not beats_cert_expiration_date.valid_at.check_period
 
 - name: Print the beats certificate renew message
   debug:
     msg: |
-      Your beats certificate will expire in {{ beats_cert_expiration_days }}.
+      Your beats certificate will expire before {{ beats_cert_expiration_buffer }}.
       Ansible will renew it.
-  when: beats_cert_expiration_days is defined and beats_cert_expiration_days | int <= beats_cert_expiration_buffer | int
+  when: beats_cert_expiration_date.skipped is not defined and not beats_cert_expiration_date.valid_at.check_period
 
 - name: Backup beats certs then remove
   when: "'renew_beats_cert' in ansible_run_tags or 'renew_ca' in ansible_run_tags or beats_cert_will_expire_soon | bool"

roles/elasticsearch/handlers/main.yml

@@ -4,6 +4,7 @@
   service:
     name: elasticsearch
     state: restarted
+    daemon_reload: yes
   when: elasticsearch_enable | bool
 
 - name: Restart kibana if available for elasticsearch certificates

roles/elasticsearch/tasks/elasticsearch-security.yml

@@ -32,13 +32,13 @@
     elasticstack_ca_will_expire_soon: true
   when: >
     inventory_hostname == elasticstack_ca and
-    elasticsearch_cert_expiration_days is defined and
-    elasticsearch_cert_expiration_days | int <= elasticsearch_cert_expiration_buffer | int
+    elasticstack_ca_expiration_days is defined and
+    elasticstack_ca_expiration_days | int <= elasticstack_ca_expiration_buffer | int
 
 - name: Print the ca renew message
   debug:
     msg: |
-      Your ca will expire in {{ elasticstack_ca_expiration_days }}.
+      Your ca will expire in {{ elasticstack_ca_expiration_days }} days.
       Ansible will renew it and all elastic stack certificates
   when: >
     inventory_hostname == elasticstack_ca and
@@ -107,24 +107,17 @@
     path: "/etc/elasticsearch/certs/{{ ansible_hostname }}.p12"
   register: elasticsearch_cert_exists
 
-- name: Get the elasticsearch certificate expiration date # noqa: risky-shell-pipe
-  shell: >-
-    if test -v BASH; then set -o pipefail; fi;
-    openssl pkcs12
-    -in "/etc/elasticsearch/certs/{{ ansible_hostname }}.p12"
-    -nodes
-    -passin pass:"{{ elasticsearch_tls_key_passphrase }}" |
-    openssl x509 -noout -enddate |
-    awk -F'=' '{print $2}'
-  register: elasticsearch_cert_expiration_date
-  changed_when: false
-  no_log: true
+- name: Get the elasticsearch certificate expiration date
+  cert_info:
+    path: "/etc/elasticsearch/certs/{{ ansible_hostname }}.p12"
+    passphrase: "{{ elasticsearch_tls_key_passphrase | default(omit, true) }}"
+  register: elasticsearch_cert_infos
   when: elasticsearch_cert_exists.stat.exists | bool
 
 - name: Set the elasticsearch certificate expiration date in days
   set_fact:
-    elasticsearch_cert_expiration_days: "{{ ((elasticsearch_cert_expiration_date.stdout | to_datetime('%b %d %H:%M:%S %Y %Z')) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
-  when: elasticsearch_cert_expiration_date.skipped is not defined
+    elasticsearch_cert_expiration_days: "{{ ((elasticsearch_cert_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
+  when: elasticsearch_cert_infos.skipped is not defined
 
 - name: Set elasticsearch certificate will expire soon to true
   set_fact:
@@ -134,9 +127,9 @@
 - name: Print the elasticsearch certificate renew message
   debug:
     msg: |
-      Your elasticsearch certificate will expire in {{ elasticsearch_cert_expiration_days }}.
+      Your elasticsearch certificate will expire in {{ elasticsearch_cert_expiration_days }} days.
       Ansible will renew it.
-  when: elasticsearch_cert_expiration_day is defined and elasticstack_ca_expiration_days | int <= elasticstack_ca_expiration_buffer | int
+  when: elasticsearch_cert_expiration_days is defined and elasticsearch_cert_expiration_days | int <= elasticsearch_cert_expiration_buffer | int
 
 - name: Backup elasticsearch certs on node then remove
   when: "'renew_es_cert' in ansible_run_tags or 'renew_ca' in ansible_run_tags or elasticsearch_cert_will_expire_soon | bool"
@@ -532,6 +525,7 @@
 - name: Wait for all instances to start
   include_tasks: wait_for_instance.yml
   loop: "{{ groups['elasticsearch'] }}"
+  tags: notest
 
 - name: Force all notified handlers to run at this point, not waiting for normal sync points
   ansible.builtin.meta: flush_handlers
@@ -544,6 +538,7 @@
   include_tasks: wait_for_instance.yml
   loop: "{{ groups['elasticsearch'] }}"
   tags:
+    - notest
     - certificates
     - renew_ca
     - renew_es_cert

roles/kibana/tasks/kibana-security.yml

@@ -24,24 +24,16 @@
     path: "/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12"
   register: kibana_cert_exists
 
-- name: Get the kibana certificate expiration date # noqa: risky-shell-pipe
-  shell: >-
-    if test -v BASH; then set -o pipefail; fi;
-    openssl pkcs12
-    -in '/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12'
-    -nodes -passin pass:'{{ kibana_tls_key_passphrase }}' |
-    openssl x509 -noout -enddate |
-    awk -F'=' '{print $2}'
+- name: Get the kibana certificate expiration date
+  cert_info:
+    path: "/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12"
+    passphrase: "{{ kibana_tls_key_passphrase | default(omit, true) }}"
   register: kibana_cert_expiration_date
-  args:
-    executable: /bin/bash
-  changed_when: false
-  no_log: true
   when: kibana_cert_exists.stat.exists | bool
 
 - name: Set the kibana certificate expiration date in days
   set_fact:
-    kibana_cert_expiration_days: "{{ ((kibana_cert_expiration_date.stdout | to_datetime('%b %d %H:%M:%S %Y %Z')) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
+    kibana_cert_expiration_days: "{{ ((kibana_cert_expiration_date.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
   when: kibana_cert_expiration_date.skipped is not defined
 
 - name: Set kibana certificate will expire soon to true
@@ -52,7 +44,7 @@
 - name: Print the kibana certificate renew message
   debug:
     msg: |
-      Your kibana certificate will expire in {{ kibana_cert_expiration_days }}.
+      Your kibana certificate will expire in {{ kibana_cert_expiration_days }} days.
       Ansible will renew it.
   when: kibana_cert_expiration_days is defined and kibana_cert_expiration_days | int <= kibana_cert_expiration_buffer | int
 

roles/kibana/tasks/main.yml

@@ -84,9 +84,10 @@
   when: kibana_enable | bool
 
 # the following is useful when running tests or extra tasks that need to
-# have Kibana running
+# have Kibana running. Escape it on Rocky8, because it gets time out with Elastic 8
 
 - name: Wait for Kibana to start
   wait_for:
     host: localhost
     port: 5601
+  tags: notest

roles/logstash/tasks/logstash-security.yml

@@ -26,40 +26,29 @@
     path: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12"
   register: logstash_cert_exists
 
-- name: Get the logstash certificate expiration date # noqa: risky-shell-pipe
-  shell: >-
-    if test -v BASH; then set -o pipefail; fi;
-    openssl pkcs12
-    -in '{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12'
-    -nodes
-    -passin pass:'{{ logstash_tls_key_passphrase }}' |
-    openssl x509
-    -noout
-    -enddate |
-    awk -F'=' '{print $2}'
+- name: Get the logstash certificate expiration date
+  cert_info:
+    path: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12"
+    passphrase: "{{ logstash_tls_key_passphrase | default(omit, true) }}"
   register: logstash_cert_expiration_date
-  args:
-    executable: /bin/bash
-  changed_when: false
-  no_log: true
   when: logstash_cert_exists.stat.exists | bool
 
 - name: Set the logstash certificate expiration date in days
   set_fact:
-    logstash_cert_expiration_days: "{{ ((logstash_cert_expiration_date.stdout | to_datetime('%b %d %H:%M:%S %Y %Z')) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
+    logstash_cert_expiration_days: "{{ ((logstash_cert_expiration_date.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
   when: logstash_cert_expiration_date.skipped is not defined
 
 - name: Set logstash certificate will expire soon to true
   set_fact:
     logstash_cert_will_expire_soon: true
-  when: kiban_cert_expiration_days is defined and logstash_cert_expiration_days | int <= logstash_cert_expiration_buffer | int
+  when: logstash_cert_expiration_days is defined and logstash_cert_expiration_days | int <= logstash_cert_expiration_buffer | int
 
 - name: Print the logstash certificate renew message
   debug:
     msg: |
-      Your logstash certificate will expire in {{ logstash_cert_expiration_days }}.
+      Your logstash certificate will expire in {{ logstash_cert_expiration_days }} days.
       Ansible will renew it.
-  when: logstash_cert_expiration_days is defined and logstash_cert_expiration_days | int <= logstash_expiration_buffer | int
+  when: logstash_cert_expiration_days is defined and logstash_cert_expiration_days | int <= logstash_cert_expiration_buffer | int
 
 - name: Backup logstash certs then remove
   when: "'renew_logstash_cert' in ansible_run_tags or 'renew_ca' in ansible_run_tags"
@@ -225,6 +214,18 @@
     - renew_logstash_cert
 
 - name: Copy the certificate to logstash node
+  copy:
+    src: "/tmp/{{ ansible_hostname }}-ls.p12"
+    dest: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12"
+    owner: root
+    group: logstash
+    mode: 0640
+  tags:
+    - certificates
+    - renew_ca
+    - renew_logstash_cert
+
+- name: Put the certificate in keystore
   copy:
     src: "/tmp/{{ ansible_hostname }}-ls.p12"
     dest: "{{ logstash_certs_dir }}/keystore.pfx"