diff --git a/.config/ansible-lint.yml b/.config/ansible-lint.yml index ac2c7bd6..b3272bf3 100644 --- a/.config/ansible-lint.yml +++ b/.config/ansible-lint.yml @@ -7,7 +7,6 @@ warn_list: - key-order[task] # Ensure specific order of keys in mappings. - name[casing] - 'risky-shell-pipe' - - no-handler # backup of old certificates - var-naming[no-role-prefix] # remove when https://github.com/ansible/ansible-lint/discussions/3451 is done skip_list: - '106' @@ -16,3 +15,4 @@ skip_list: - 'line-length' - 'package-latest' - yaml # Violations reported by yamllint. + - no-handler # backup of old certificates diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md index 1675e789..6b256761 100644 --- a/docs/logstash-pipelines.md +++ b/docs/logstash-pipelines.md @@ -205,7 +205,7 @@ Every Output can have a `congestion:` option with a numerical value. If the Redi ### Unsafe shutdown ### -If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. If you want better controll over which pipeline is allowed to shutdown unsafely, there are `ansible_input_unsafe_shutdown`and `ansible_forwarder_unsafe_shutdown` for default pipelines. And every pipeline has it's own `unsafe_shutdown` setting. All three default to the value of `logstash_pipeline_unsafe_shutdown` which by itself defaults to `false`. +If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. ## Caveats ## diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index e82ed39b..ded6555a 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -13,9 +13,10 @@ elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticsearch_heap: "1" + elasticsearch_heap: "2" elasticstack_full_stack: true logstash_pipeline_unsafe_shutdown: true + logstash_password_hash: false beats_filebeat_syslog_udp: true beats_filebeat_syslog_tcp: true beats_filebeat_modules: @@ -28,10 +29,13 @@ elasticstack_rpm_workaround: true when: ansible_os_family == 'RedHat' and ansible_distribution_major_version >= "9" - name: Update apt cache. - apt: update_cache=yes cache_valid_time=600 + ansible.builtin.apt: + update_cache: yes + cache_valid_time: 600 + changed_when: false when: ansible_os_family == 'Debian' - name: Install dependencies - package: + ansible.builtin.package: name: - curl - name: Include Redis diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index 11e5a015..abceef3f 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -7,10 +7,6 @@ elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords tasks: - - name: Give some time for tools to connect - ansible.builtin.wait_for: - timeout: 120 - - name: Run Logstash syntax check ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t" when: "'logstash' in group_names" diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 00050d10..383111a2 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -24,7 +24,7 @@ - name: Set the ca expiration date in days ansible.builtin.set_fact: - elasticstack_ca_expiration_days: "{{ (( elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}" + elasticstack_ca_expiration_days: "{{ ((elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}" when: inventory_hostname == elasticstack_ca and elasticstack_ca_infos.skipped is not defined - name: Set ca will expire soon to true diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 74803121..a8751815 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -148,8 +148,8 @@ mode: "2750" when: item.create | bool loop: - - {create: "{{elasticsearch_create_logpath}}", path: "{{ elasticsearch_logpath }}" } - - {create: "{{elasticsearch_create_datapath}}", path: "{{ elasticsearch_datapath }}" } + - {create: "{{ elasticsearch_create_logpath }}", path: "{{ elasticsearch_logpath }}" } + - {create: "{{ elasticsearch_create_datapath }}", path: "{{ elasticsearch_datapath }}" } - name: Import Tasks elasticsearch-security.yml ansible.builtin.import_tasks: elasticsearch-security.yml diff --git a/roles/elasticsearch/templates/jvm.options.j2 b/roles/elasticsearch/templates/jvm.options.j2 index 9e663ca5..f85b3deb 100644 --- a/roles/elasticsearch/templates/jvm.options.j2 +++ b/roles/elasticsearch/templates/jvm.options.j2 @@ -73,15 +73,15 @@ ################################################################ ## GC configuration -{% if elasticstack_release is version('7.6.0', '<') %} --XX:+UseConcMarkSweepGC --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly -{% else %} -8-13:-XX:+UseConcMarkSweepGC -8-13:-XX:CMSInitiatingOccupancyFraction=75 -8-13:-XX:+UseCMSInitiatingOccupancyOnly -{% endif %} +#{% if elasticstack_release is version('7.6.0', '<') %} +##-XX:+UseConcMarkSweepGC +##-XX:CMSInitiatingOccupancyFraction=75 +##-XX:+UseCMSInitiatingOccupancyOnly +#{% else %} +#8-13:-XX:+UseConcMarkSweepGC +#8-13:-XX:CMSInitiatingOccupancyFraction=75 +#8-13:-XX:+UseCMSInitiatingOccupancyOnly +#{% endif %} ## G1GC Configuration # NOTE: G1 GC is only supported on JDK version 10 or later diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index ad1e0279..ac876fba 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -17,7 +17,6 @@ logstash_config_path_logs: /var/log/logstash # pipeline configuration # logstash_manage_pipelines: true logstash_queue_type: persisted -logstash_pipeline_unsafe_shutdown: false # this will deactivate all pipeline management logstash_no_pipelines: false diff --git a/roles/logstash/templates/logstash.yml.j2 b/roles/logstash/templates/logstash.yml.j2 index 2aaada6c..20266856 100644 --- a/roles/logstash/templates/logstash.yml.j2 +++ b/roles/logstash/templates/logstash.yml.j2 @@ -13,6 +13,9 @@ http.port: {{ logstash_http_port }} {% if logstash_global_ecs is defined %} pipeline.ecs_compatibility: {{ logstash_global_ecs }} {% endif %} +{% if pipeline.unsafe_shutdown is defined %} +pipeline.unsafe_shutdown: {{ logstash_pipeline_unsafe_shutdown }} +{% endif %} {% if logstash_legacy_monitoring | bool and elasticstack_full_stack | bool and elasticstack_variant == "elastic" and elasticstack_release | int < 8 %} xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: [ {% for host in logstash_elasticsearch %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %} ] diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index daa57a1c..901088e6 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -19,14 +19,10 @@ # Default beat input # # Autoconfigured Redis outputs: input -- pipeline - id: ansible-input - unsafe_shutdown: {{ ansible_input_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path - config: "/etc/logstash/conf.d/ansible-input/*.conf" - queue - type: {{ logstash_input_queue_type }} - max_bytes: {{ logstash_input_queue_max_bytes }} +- pipeline.id: ansible-input + path.config: "/etc/logstash/conf.d/ansible-input/*.conf" + queue.type: {{ logstash_input_queue_type }} + queue.max_bytes: {{ logstash_input_queue_max_bytes }} {% endif %} {% if logstash_elasticsearch_output | bool %} @@ -34,14 +30,10 @@ # Default elasticsearch output # # Autoconfigured Redis input: forwarder -- pipeline - id: ansible-forwarder - unsafe_shutdown: {{ ansible_forwarder_unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path - config: "/etc/logstash/conf.d/ansible-forwarder/*.conf" - queue - type: {{ logstash_forwarder_queue_type }} - max_bytes: {{ logstash_forwarder_queue_max_bytes }} +- pipeline.id: ansible-forwarder + path.config: "/etc/logstash/conf.d/ansible-forwarder/*.conf" + queue.type: {{ logstash_forwarder_queue_type }} + queue.max_bytes: {{ logstash_forwarder_queue_max_bytes }} {% endif %} {% if logstash_pipelines is defined %} @@ -72,14 +64,10 @@ {% endif %} -- pipeline - id: {{ item.name }} - unsafe_shutdown: {{ item.unsafe_shutdown | default({{ logstash_pipeline_unsafe_shutdown }}) }} - path - config: "/etc/logstash/conf.d/{{ item.name }}/*.conf" - queue - type: {{ item.queue_type | default('memory') }} - max_bytes: {{ item.queue_max_bytes | default('1gb') }} +- pipeline.id: {{ item.name }} + path.config: "/etc/logstash/conf.d/{{ item.name }}/*.conf" + queue.type: {{ item.queue_type | default('memory') }} + queue.max_bytes: {{ item.queue_max_bytes | default('1gb') }} {% endfor %} {% endif %}