Add option to stop ML jobs and set cluster allocation mode. Update docs

Author: ivaeri-norkart

docs/role-elasticsearch.md

@@ -9,6 +9,9 @@ If you use the role to set up security you, can use its CA to create certificate
 
 Please note that setting `elasticsearch_bootstrap_pw` as variable will only take effect when initialising Elasticsearch. Changes after starting elasticsearch for the first time will not change the bootstrap password for the instance and will lead to breaking tests.
 
+The role can perform a rolling upgrade of an elasticsearch cluster. Nodes will be upgraded in inventory order. If you are using data tires, you're ansible inventory should be sorted acording to data tiers.  Set `any_errors_fatal: true` in your playbook to abort if any errors occur during the upgrade.
+
+
 Role Variables
 --------------
 
@@ -54,6 +57,9 @@ This variable activates a workaround to start on systems that have certain harde
 * *elasticsearch_seed_hosts*: Set elasticsearch seed hosts
 * *elasticsearch_security_enrollment*: Controls enrollment (of nodes and Kibana) to a local node that’s been autoconfigured for security.
 
+*elasticsearch_upgrade_routing_mode*: Set `cluster.routing.allocation.enable` during rolling upgrade. (default: `none`)
+*elasticsearch_upgrade_stop_ml*: Stop the tasks associated with active machine learning jobs and datafeeds during rolling upgrade. (default: `false`)
+
 The following variable was only integrated to speed up upgrades of non-production clusters. Use with caution and at your own risk:
 
 * *elasticsearch_unsafe_upgrade_restart*: This will still perform rolling upgrades, but will first update the package and then restart the service. In contrast the default behaviour is to stop the service, do the upgrade and then start again. (default: `false`)

roles/elasticsearch/defaults/main.yml

@@ -37,6 +37,10 @@ elasticsearch_cert_expiration_buffer: 30
 elasticsearch_cert_will_expire_soon: false
 elasticsearch_ssl_verification_mode: full
 
+# Upgrade options
+elasticsearch_upgrade_routing_mode: "none"
+elasticsearch_upgrade_stop_ml: false
+
 # use this only for non-prod environments and at your own risk!
 elasticsearch_unsafe_upgrade_restart: false
 

roles/elasticsearch/tasks/elasticsearch-rolling-start.yml

@@ -55,3 +55,14 @@
 # Don't continue the play unless cluster health is OK
 - name: Cluster health check
   ansible.builtin.include_tasks: elasticsearch-wait-for-cluster-health.yml
+
+- name: Restart ML jobs
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_ml/set_upgrade_mode?enabled=false"
+    method: POST
+    user: elastic
+    password: "{{ elasticstack_password.stdout }}"
+    validate_certs: no
+  failed_when: false
+  when:
+    - elasticsearch_upgrade_stop_ml | bool

roles/elasticsearch/tasks/elasticsearch-rolling-stop.yml

@@ -21,12 +21,23 @@
   ansible.builtin.include_tasks: elasticsearch-wait-for-cluster-health.yml
 
 
-# Disabling shard allocation right after enabling it seems redundant. Please see above for details.
+- name: Stop ML jobs while upgrading
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_ml/set_upgrade_mode?enabled=true"
+    method: POST
+    user: elastic
+    password: "{{ elasticstack_password.stdout }}"
+    validate_certs: no
+  failed_when: false
+  run_once: true
+  when:
+    - elasticsearch_upgrade_stop_ml | bool
+
 - name: Disable shard allocation for the cluster
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings"
     method: PUT
-    body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}'
+    body: "{ \"persistent\": { \"cluster.routing.allocation.enable\": \"{{ elasticsearch_upgrade_routing_mode }}\" } }"
     body_format: json
     user: elastic
     password: "{{ elasticstack_password.stdout }}"

roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml

@@ -64,7 +64,6 @@
       notify:
         - Restart Elasticsearch
 
-
 - name: Be careful about upgrade when Elasticsearch is running
   when:
     - elasticsearch_running.status.ActiveState == "active"