From 62b68b114d42e6e607f75490a217c7896bbd40b4 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 11:28:36 +0200 Subject: [PATCH 1/7] Fix YAML/Jinja2 syntax. Simple as that fixes #242 --- roles/logstash/templates/pipelines.yml.j2 | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index daa57a1c..1a36ad8c 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -19,12 +19,12 @@ # Default beat input # # Autoconfigured Redis outputs: input -- pipeline +- pipeline: id: ansible-input - unsafe_shutdown: {{ ansible_input_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path + unsafe_shutdown: {{ ansible_input_unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} + path: config: "/etc/logstash/conf.d/ansible-input/*.conf" - queue + queue: type: {{ logstash_input_queue_type }} max_bytes: {{ logstash_input_queue_max_bytes }} @@ -34,12 +34,12 @@ # Default elasticsearch output # # Autoconfigured Redis input: forwarder -- pipeline +- pipeline: id: ansible-forwarder - unsafe_shutdown: {{ ansible_forwarder_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path + unsafe_shutdown: {{ ansible_forwarder_unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} + path: config: "/etc/logstash/conf.d/ansible-forwarder/*.conf" - queue + queue: type: {{ logstash_forwarder_queue_type }} max_bytes: {{ logstash_forwarder_queue_max_bytes }} @@ -72,12 +72,12 @@ {% endif %} -- pipeline +- pipeline: id: {{ item.name }} - unsafe_shutdown: {{ item.unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path + unsafe_shutdown: {{ item.unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} + path: config: "/etc/logstash/conf.d/{{ item.name }}/*.conf" - queue + queue: type: {{ item.queue_type | default('memory') }} max_bytes: {{ item.queue_max_bytes | default('1gb') }} From e18849d751dc7ebffc1e3f16eb899ae7b4b0d9cf Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 13:04:44 +0200 Subject: [PATCH 2/7] Try with more elasticsearch heap --- molecule/elasticstack_default/converge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index e82ed39b..dda936b7 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -13,7 +13,7 @@ elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticsearch_heap: "1" + elasticsearch_heap: "2" elasticstack_full_stack: true logstash_pipeline_unsafe_shutdown: true beats_filebeat_syslog_udp: true From a5c88d449ffb17a59d0c5afea2d85890805d8b3c Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 15:48:28 +0200 Subject: [PATCH 3/7] Fixed syntax for syntax check TIL: * Logstash can read YAML. Logstash config check CAN NOT. * Logstash documentation lists per-pipeline and global options in the same list. Config check does not allow to mix them * Sometimes it just sucks to be alive Damn, that cost a lot of time and local debugging in test hosts. --- docs/logstash-pipelines.md | 2 +- molecule/elasticstack_default/converge.yml | 1 + molecule/elasticstack_default/verify.yml | 4 --- roles/logstash/defaults/main.yml | 1 - roles/logstash/templates/logstash.yml.j2 | 3 ++ roles/logstash/templates/pipelines.yml.j2 | 36 ++++++++-------------- 6 files changed, 17 insertions(+), 30 deletions(-) diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md index 1675e789..6b256761 100644 --- a/docs/logstash-pipelines.md +++ b/docs/logstash-pipelines.md @@ -205,7 +205,7 @@ Every Output can have a `congestion:` option with a numerical value. If the Redi ### Unsafe shutdown ### -If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. If you want better controll over which pipeline is allowed to shutdown unsafely, there are `ansible_input_unsafe_shutdown`and `ansible_forwarder_unsafe_shutdown` for default pipelines. And every pipeline has it's own `unsafe_shutdown` setting. All three default to the value of `logstash_pipeline_unsafe_shutdown` which by itself defaults to `false`. +If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. ## Caveats ## diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index dda936b7..d192a5e0 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -16,6 +16,7 @@ elasticsearch_heap: "2" elasticstack_full_stack: true logstash_pipeline_unsafe_shutdown: true + logstash_password_hash: false beats_filebeat_syslog_udp: true beats_filebeat_syslog_tcp: true beats_filebeat_modules: diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index 11e5a015..abceef3f 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -7,10 +7,6 @@ elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords tasks: - - name: Give some time for tools to connect - ansible.builtin.wait_for: - timeout: 120 - - name: Run Logstash syntax check ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t" when: "'logstash' in group_names" diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index ad1e0279..ac876fba 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -17,7 +17,6 @@ logstash_config_path_logs: /var/log/logstash # pipeline configuration # logstash_manage_pipelines: true logstash_queue_type: persisted -logstash_pipeline_unsafe_shutdown: false # this will deactivate all pipeline management logstash_no_pipelines: false diff --git a/roles/logstash/templates/logstash.yml.j2 b/roles/logstash/templates/logstash.yml.j2 index 2aaada6c..20266856 100644 --- a/roles/logstash/templates/logstash.yml.j2 +++ b/roles/logstash/templates/logstash.yml.j2 @@ -13,6 +13,9 @@ http.port: {{ logstash_http_port }} {% if logstash_global_ecs is defined %} pipeline.ecs_compatibility: {{ logstash_global_ecs }} {% endif %} +{% if pipeline.unsafe_shutdown is defined %} +pipeline.unsafe_shutdown: {{ logstash_pipeline_unsafe_shutdown }} +{% endif %} {% if logstash_legacy_monitoring | bool and elasticstack_full_stack | bool and elasticstack_variant == "elastic" and elasticstack_release | int < 8 %} xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: [ {% for host in logstash_elasticsearch %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %} ] diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index 1a36ad8c..901088e6 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -19,14 +19,10 @@ # Default beat input # # Autoconfigured Redis outputs: input -- pipeline: - id: ansible-input - unsafe_shutdown: {{ ansible_input_unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} - path: - config: "/etc/logstash/conf.d/ansible-input/*.conf" - queue: - type: {{ logstash_input_queue_type }} - max_bytes: {{ logstash_input_queue_max_bytes }} +- pipeline.id: ansible-input + path.config: "/etc/logstash/conf.d/ansible-input/*.conf" + queue.type: {{ logstash_input_queue_type }} + queue.max_bytes: {{ logstash_input_queue_max_bytes }} {% endif %} {% if logstash_elasticsearch_output | bool %} @@ -34,14 +30,10 @@ # Default elasticsearch output # # Autoconfigured Redis input: forwarder -- pipeline: - id: ansible-forwarder - unsafe_shutdown: {{ ansible_forwarder_unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} - path: - config: "/etc/logstash/conf.d/ansible-forwarder/*.conf" - queue: - type: {{ logstash_forwarder_queue_type }} - max_bytes: {{ logstash_forwarder_queue_max_bytes }} +- pipeline.id: ansible-forwarder + path.config: "/etc/logstash/conf.d/ansible-forwarder/*.conf" + queue.type: {{ logstash_forwarder_queue_type }} + queue.max_bytes: {{ logstash_forwarder_queue_max_bytes }} {% endif %} {% if logstash_pipelines is defined %} @@ -72,14 +64,10 @@ {% endif %} -- pipeline: - id: {{ item.name }} - unsafe_shutdown: {{ item.unsafe_shutdown | default( logstash_pipeline_unsafe_shutdown ) }} - path: - config: "/etc/logstash/conf.d/{{ item.name }}/*.conf" - queue: - type: {{ item.queue_type | default('memory') }} - max_bytes: {{ item.queue_max_bytes | default('1gb') }} +- pipeline.id: {{ item.name }} + path.config: "/etc/logstash/conf.d/{{ item.name }}/*.conf" + queue.type: {{ item.queue_type | default('memory') }} + queue.max_bytes: {{ item.queue_max_bytes | default('1gb') }} {% endfor %} {% endif %} From 08199353e2858b1873de984877ee8337ba9463b9 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 16:19:52 +0200 Subject: [PATCH 4/7] Quick workaround for jvm.options This is a more serious problem. I opened a separate issue for it: https://github.com/NETWAYS/ansible-collection-elasticstack/issues/246 --- roles/elasticsearch/templates/jvm.options.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/elasticsearch/templates/jvm.options.j2 b/roles/elasticsearch/templates/jvm.options.j2 index 9e663ca5..f85b3deb 100644 --- a/roles/elasticsearch/templates/jvm.options.j2 +++ b/roles/elasticsearch/templates/jvm.options.j2 @@ -73,15 +73,15 @@ ################################################################ ## GC configuration -{% if elasticstack_release is version('7.6.0', '<') %} --XX:+UseConcMarkSweepGC --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly -{% else %} -8-13:-XX:+UseConcMarkSweepGC -8-13:-XX:CMSInitiatingOccupancyFraction=75 -8-13:-XX:+UseCMSInitiatingOccupancyOnly -{% endif %} +#{% if elasticstack_release is version('7.6.0', '<') %} +##-XX:+UseConcMarkSweepGC +##-XX:CMSInitiatingOccupancyFraction=75 +##-XX:+UseCMSInitiatingOccupancyOnly +#{% else %} +#8-13:-XX:+UseConcMarkSweepGC +#8-13:-XX:CMSInitiatingOccupancyFraction=75 +#8-13:-XX:+UseCMSInitiatingOccupancyOnly +#{% endif %} ## G1GC Configuration # NOTE: G1 GC is only supported on JDK version 10 or later From b5f67facb1e2c89d217307ebd6ca46c91fd3c8b8 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 16:45:36 +0200 Subject: [PATCH 5/7] Disable warnings in linter Actually we won't be able to change that easily. So I disabled the warning. I know, I opted for warning in the first place but seeing all the warnings without a way to solve them, I changed my mind. --- .config/ansible-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/ansible-lint.yml b/.config/ansible-lint.yml index ac2c7bd6..b3272bf3 100644 --- a/.config/ansible-lint.yml +++ b/.config/ansible-lint.yml @@ -7,7 +7,6 @@ warn_list: - key-order[task] # Ensure specific order of keys in mappings. - name[casing] - 'risky-shell-pipe' - - no-handler # backup of old certificates - var-naming[no-role-prefix] # remove when https://github.com/ansible/ansible-lint/discussions/3451 is done skip_list: - '106' @@ -16,3 +15,4 @@ skip_list: - 'line-length' - 'package-latest' - yaml # Violations reported by yamllint. + - no-handler # backup of old certificates From c197e22383f4ec60d470fc804d138053c1449fd2 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 16:48:45 +0200 Subject: [PATCH 6/7] Apply Jinja suggestions from linter --- roles/elasticsearch/tasks/elasticsearch-security.yml | 2 +- roles/elasticsearch/tasks/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 00050d10..383111a2 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -24,7 +24,7 @@ - name: Set the ca expiration date in days ansible.builtin.set_fact: - elasticstack_ca_expiration_days: "{{ (( elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}" + elasticstack_ca_expiration_days: "{{ ((elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}" when: inventory_hostname == elasticstack_ca and elasticstack_ca_infos.skipped is not defined - name: Set ca will expire soon to true diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 74803121..a8751815 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -148,8 +148,8 @@ mode: "2750" when: item.create | bool loop: - - {create: "{{elasticsearch_create_logpath}}", path: "{{ elasticsearch_logpath }}" } - - {create: "{{elasticsearch_create_datapath}}", path: "{{ elasticsearch_datapath }}" } + - {create: "{{ elasticsearch_create_logpath }}", path: "{{ elasticsearch_logpath }}" } + - {create: "{{ elasticsearch_create_datapath }}", path: "{{ elasticsearch_datapath }}" } - name: Import Tasks elasticsearch-security.yml ansible.builtin.import_tasks: elasticsearch-security.yml From 7a865de7f562afa02071e7eeb1b097631ededd1a Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 23 Aug 2023 17:01:33 +0200 Subject: [PATCH 7/7] Disable change for apt refresh It breaks idempotence --- molecule/elasticstack_default/converge.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index d192a5e0..ded6555a 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -29,10 +29,13 @@ elasticstack_rpm_workaround: true when: ansible_os_family == 'RedHat' and ansible_distribution_major_version >= "9" - name: Update apt cache. - apt: update_cache=yes cache_valid_time=600 + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false when: ansible_os_family == 'Debian' - name: Install dependencies - package: + ansible.builtin.package: name: - curl - name: Include Redis