diff --git a/chaoslib/pumba/network_chaos/network_chaos.yml b/chaoslib/pumba/network_chaos/network_chaos.yml index 07b6f4a6547..9daa1a792fa 100644 --- a/chaoslib/pumba/network_chaos/network_chaos.yml +++ b/chaoslib/pumba/network_chaos/network_chaos.yml @@ -93,8 +93,9 @@ kubectl delete -f /chaoslib/pumba/network_chaos/pumba_kube.yml -n {{ a_ns }} args: executable: /bin/bash + when: pumba_deploy_result.rc == 0 - - name: Confirm that the pumba job is deleted successfully + - name: Confirm that the pumba job is not present shell: > kubectl get pods -l job-name=pumba-netem-{{ run_id }} --no-headers -n {{ a_ns }} args: @@ -103,5 +104,4 @@ until: "'No resources found' in result.stderr" delay: 5 retries: 60 - - when: "pumba_deploy_result.rc is defined and pumba_deploy_result.rc == 0" + when: "pumba_deploy_result is defined" diff --git a/experiments/generic/pod_network_latency/pod_network_latency_k8s_job.yml b/experiments/generic/pod_network_latency/pod_network_latency_k8s_job.yml index 3ae8866bca9..2f157badc46 100644 --- a/experiments/generic/pod_network_latency/pod_network_latency_k8s_job.yml +++ b/experiments/generic/pod_network_latency/pod_network_latency_k8s_job.yml @@ -41,7 +41,7 @@ spec: # provide lib image - name: LIB_IMAGE - value: 'gaiaadm/pumba:0.4.8' + value: 'gaiaadm/pumba:0.6.5' - name: NETWORK_LATENCY value: '60000' # in ms diff --git a/experiments/kafka/kafka-broker-network-latency/chaosutil.j2 b/experiments/kafka/kafka-broker-network-latency/chaosutil.j2 new file mode 100644 index 00000000000..090dc947340 --- /dev/null +++ b/experiments/kafka/kafka-broker-network-latency/chaosutil.j2 @@ -0,0 +1,4 @@ +{% if c_lib is defined and c_lib == 'pumba' %} + c_util: /chaoslib/pumba/network_chaos/network_chaos.yml +{% endif %} + diff --git a/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-logic.yml b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-logic.yml new file mode 100644 index 00000000000..5b3f539408a --- /dev/null +++ b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-logic.yml @@ -0,0 +1,118 @@ +--- +- hosts: localhost + connection: local + + vars: + c_experiment: "kafka-broker-network-latency" + c_duration: "{{ lookup('env','TOTAL_CHAOS_DURATION') }}" + c_container: "{{ lookup('env','TARGET_CONTAINER') }}" + n_latency: "{{ lookup('env','NETWORK_LATENCY') }}" + n_interface: "{{ lookup('env','NETWORK_INTERFACE') }}" + c_lib: "{{ lookup('env','LIB') }}" + lib_image: "{{ lookup('env','LIB_IMAGE') }}" + kafka_instance: "{{ lookup('env','KAFKA_INSTANCE_NAME') }}" + kafka_ns: "{{ lookup('env','KAFKA_NAMESPACE') }}" + kafka_label: "{{ lookup('env','KAFKA_LABEL') }}" + kafka_kind: "{{ lookup('env','KAFKA_KIND') }}" + kafka_broker: "{{ lookup('env','KAFKA_BROKER') }}" + kafka_stream: "{{ lookup('env','KAFKA_LIVENESS_STREAM') }}" + kafka_liveness_image: "{{ lookup('env','KAFKA_LIVENESS_IMAGE') }}" + kafka_consumer_timeout: "{{ lookup('env','KAFKA_CONSUMER_TIMEOUT') }}" + kafka_service: "{{ lookup('env','KAFKA_SERVICE') }}" + kafka_port: "{{ lookup('env','KAFKA_PORT') }}" + kafka_replication_factor: "{{ lookup('env','KAFKA_REPLICATION_FACTOR') }}" + zk_ns: "{{ lookup('env','ZOOKEEPER_NAMESPACE') }}" + zk_label: "{{ lookup('env','ZOOKEEPER_LABEL') }}" + zk_service: "{{ lookup('env','ZOOKEEPER_SERVICE') }}" + zk_port: "{{ lookup('env','ZOOKEEPER_PORT') }}" + + tasks: + - block: + + - include: kafka-broker-network-latency-ansible-prerequisites.yml + + - include_vars: + file: chaosutil.yml + + ## GENERATE EXP RESULT NAME + - block: + + - name: Construct chaos result name (experiment_name) + set_fact: + c_experiment: "{{ lookup('env','CHAOSENGINE') }}-{{ c_experiment }}" + + when: lookup('env','CHAOSENGINE') + + ## RECORD START-OF-EXPERIMENT IN LITMUSCHAOS RESULT CR + - include_tasks: /utils/runtime/update_chaos_result_resource.yml + vars: + status: 'SOT' + namespace: "{{ kafka_ns }}" + + ## PRE-CHAOS APPLICATION LIVENESS CHECK + + - name: Verify that the Kafka cluster is healthy + include_tasks: "/utils/apps/kafka/kafka_cluster_health.yml" + vars: + delay: 1 + retries: 60 + + ## SETUP KAFKA CHAOS INFRA AND DERIVE BROKERS UNDER TEST + + - include_tasks: "{{ kafka_broker_util }}" + + ## FAULT INJECTION + + - include_tasks: "{{ c_util }}" + vars: + a_ns: "{{ kafka_ns }}" + a_label: "{{ kafka_label }}" + app_ns: "{{ kafka_ns }}" + + # derived from the 'kafka_broker_util' task + app_pod: "{{ kafka_broker }}" + + ## POST-CHAOS APPLICATION LIVENESS CHECK + + - name: Verify that the Kafka cluster is healthy + include_tasks: "/utils/apps/kafka/kafka_cluster_health.yml" + vars: + delay: 1 + retries: 60 + + ## CHECK FOR KAFKA LIVENESS & CLEANUP + + - block: + + - name: Verify that the Kafka liveness pod (pub-sub) is uninterrupted + include_tasks: "/utils/common/status_app_pod.yml" + vars: + a_ns: "{{ kafka_ns }}" + a_label: "name=kafka-liveness" + delay: 1 + retries: 60 + + - include_tasks: "/utils/apps/kafka/kafka_liveness_cleanup.yml" + + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + flag: "pass" + + + rescue: + - set_fact: + flag: "fail" + + - name: Cleanup kafka liveness pods if present + include_tasks: "/utils/apps/kafka/kafka_liveness_cleanup.yml" + ignore_errors: true + + always: + + ## RECORD END-OF-TEST IN LITMUSCHAOS RESULT CR + - include_tasks: /utils/runtime/update_chaos_result_resource.yml + vars: + status: 'EOT' + namespace: "{{ kafka_ns }}" + diff --git a/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-prerequisites.yml b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-prerequisites.yml new file mode 100644 index 00000000000..9cce94f7a0a --- /dev/null +++ b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-prerequisites.yml @@ -0,0 +1,31 @@ +- name: Identify the chaos util to be invoked + template: + src: chaosutil.j2 + dest: chaosutil.yml + +- block: + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_liveness_stream.yml" + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/display_kafka_broker_info.yml" + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' + + when: kafka_broker is defined and kafka_broker != '' + +- block: + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_launch_stream_derive_leader_broker.yml" + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_select_broker.yml" + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' + + when: kafka_broker is not defined or kafka_broker == '' + + + diff --git a/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-k8s-job.yml b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-k8s-job.yml new file mode 100644 index 00000000000..402afb4d0d1 --- /dev/null +++ b/experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-k8s-job.yml @@ -0,0 +1,102 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + generateName: kafka-broker-network-latency- +spec: + template: + metadata: + labels: + experiment: kafka-broker-network-latency + spec: + # Placeholder that is updated by the executor for automated runs + # Provide appropriate SA (with desired permissions) if executed manually + serviceAccountName: %CHAOS_SERVICE_ACCOUNT% + restartPolicy: Never + containers: + - name: ansibletest + image: litmuschaos/ansible-runner:ci + imagePullPolicy: Always + env: + - name: ANSIBLE_STDOUT_CALLBACK + value: 'default' + + - name: KAFKA_KIND + value: 'statefulset' + + - name: KAFKA_LIVENESS_STREAM + value: 'enabled' + + - name: KAFKA_LIVENESS_IMAGE + value: 'litmuschaos/kafka-client:ci' + + # in milliseconds + - name: KAFKA_CONSUMER_TIMEOUT + value: '30000' + + # in milliseconds + - name: TOTAL_CHAOS_DURATION + value: '60000' + + - name: TARGET_CONTAINER + value: 'k8skafka' + + # in milliseconds + - name: NETWORK_LATENCY + value: '60000' + + - name: NETWORK_INTERFACE + value: 'eth0' + + - name: KAFKA_INSTANCE_NAME + value: '' + + - name: KAFKA_NAMESPACE + value: '' + + - name: KAFKA_LABEL + value: '' + + - name: KAFKA_BROKER + value: '' + + - name: KAFKA_REPLICATION_FACTOR + value: '' + + - name: KAFKA_SERVICE + value: '' + + - name: KAFKA_PORT + value: '' + + - name: ZOOKEEPER_NAMESPACE + value: '' + + - name: ZOOKEEPER_LABEL + value: '' + + - name: ZOOKEEPER_SERVICE + value: '' + + - name: ZOOKEEPER_PORT + value: '' + + ## env var that describes the library used to execute the chaos + ## supported values: pumba + - name: LIB + value: 'pumba' + + - name: LIB_IMAGE + value: 'gaiaadm/pumba:0.6.5' + + - name: CHAOSENGINE + value: '' + + - name: CHAOS_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + + command: ["/bin/bash"] + args: ["-c", "ansible-playbook ./experiments/kafka/kafka-broker-network-latency/kafka-broker-network-latency-ansible-logic.yml -vv -i /etc/ansible/hosts; exit 0"] + diff --git a/experiments/kafka/kafka-broker-network-loss/chaosutil.j2 b/experiments/kafka/kafka-broker-network-loss/chaosutil.j2 new file mode 100644 index 00000000000..090dc947340 --- /dev/null +++ b/experiments/kafka/kafka-broker-network-loss/chaosutil.j2 @@ -0,0 +1,4 @@ +{% if c_lib is defined and c_lib == 'pumba' %} + c_util: /chaoslib/pumba/network_chaos/network_chaos.yml +{% endif %} + diff --git a/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-logic.yml b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-logic.yml new file mode 100644 index 00000000000..d9311098f2b --- /dev/null +++ b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-logic.yml @@ -0,0 +1,118 @@ +--- +- hosts: localhost + connection: local + + vars: + c_experiment: "kafka-broker-network-loss" + c_duration: "{{ lookup('env','TOTAL_CHAOS_DURATION') }}" + c_container: "{{ lookup('env','TARGET_CONTAINER') }}" + n_packet_loss: "{{ lookup('env','NETWORK_PACKET_LOSS_PERCENTAGE') }}" + n_interface: "{{ lookup('env','NETWORK_INTERFACE') }}" + c_lib: "{{ lookup('env','LIB') }}" + lib_image: "{{ lookup('env','LIB_IMAGE') }}" + kafka_instance: "{{ lookup('env','KAFKA_INSTANCE_NAME') }}" + kafka_ns: "{{ lookup('env','KAFKA_NAMESPACE') }}" + kafka_label: "{{ lookup('env','KAFKA_LABEL') }}" + kafka_kind: "{{ lookup('env','KAFKA_KIND') }}" + kafka_broker: "{{ lookup('env','KAFKA_BROKER') }}" + kafka_stream: "{{ lookup('env','KAFKA_LIVENESS_STREAM') }}" + kafka_liveness_image: "{{ lookup('env','KAFKA_LIVENESS_IMAGE') }}" + kafka_consumer_timeout: "{{ lookup('env','KAFKA_CONSUMER_TIMEOUT') }}" + kafka_service: "{{ lookup('env','KAFKA_SERVICE') }}" + kafka_port: "{{ lookup('env','KAFKA_PORT') }}" + kafka_replication_factor: "{{ lookup('env','KAFKA_REPLICATION_FACTOR') }}" + zk_ns: "{{ lookup('env','ZOOKEEPER_NAMESPACE') }}" + zk_label: "{{ lookup('env','ZOOKEEPER_LABEL') }}" + zk_service: "{{ lookup('env','ZOOKEEPER_SERVICE') }}" + zk_port: "{{ lookup('env','ZOOKEEPER_PORT') }}" + + tasks: + - block: + + - include: kafka-broker-network-loss-ansible-prerequisites.yml + + - include_vars: + file: chaosutil.yml + + ## GENERATE EXP RESULT NAME + - block: + + - name: Construct chaos result name (experiment_name) + set_fact: + c_experiment: "{{ lookup('env','CHAOSENGINE') }}-{{ c_experiment }}" + + when: lookup('env','CHAOSENGINE') + + ## RECORD START-OF-EXPERIMENT IN LITMUSCHAOS RESULT CR + - include_tasks: /utils/runtime/update_chaos_result_resource.yml + vars: + status: 'SOT' + namespace: "{{ kafka_ns }}" + + ## PRE-CHAOS APPLICATION LIVENESS CHECK + + - name: Verify that the Kafka cluster is healthy + include_tasks: "/utils/apps/kafka/kafka_cluster_health.yml" + vars: + delay: 1 + retries: 60 + + ## SETUP KAFKA CHAOS INFRA AND DERIVE BROKERS UNDER TEST + + - include_tasks: "{{ kafka_broker_util }}" + + ## FAULT INJECTION + + - include_tasks: "{{ c_util }}" + vars: + a_ns: "{{ kafka_ns }}" + a_label: "{{ kafka_label }}" + app_ns: "{{ kafka_ns }}" + + # derived from the 'kafka_broker_util' task + app_pod: "{{ kafka_broker }}" + + ## POST-CHAOS APPLICATION LIVENESS CHECK + + - name: Verify that the Kafka cluster is healthy + include_tasks: "/utils/apps/kafka/kafka_cluster_health.yml" + vars: + delay: 1 + retries: 60 + + ## CHECK FOR KAFKA LIVENESS & CLEANUP + + - block: + + - name: Verify that the Kafka liveness pod (pub-sub) is uninterrupted + include_tasks: "/utils/common/status_app_pod.yml" + vars: + a_ns: "{{ kafka_ns }}" + a_label: "name=kafka-liveness" + delay: 1 + retries: 60 + + - include_tasks: "/utils/apps/kafka/kafka_liveness_cleanup.yml" + + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + flag: "pass" + + + rescue: + - set_fact: + flag: "fail" + + - name: Cleanup kafka liveness pods if present + include_tasks: "/utils/apps/kafka/kafka_liveness_cleanup.yml" + ignore_errors: true + + always: + + ## RECORD END-OF-TEST IN LITMUSCHAOS RESULT CR + - include_tasks: /utils/runtime/update_chaos_result_resource.yml + vars: + status: 'EOT' + namespace: "{{ kafka_ns }}" + diff --git a/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-prerequisites.yml b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-prerequisites.yml new file mode 100644 index 00000000000..9cce94f7a0a --- /dev/null +++ b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-prerequisites.yml @@ -0,0 +1,31 @@ +- name: Identify the chaos util to be invoked + template: + src: chaosutil.j2 + dest: chaosutil.yml + +- block: + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_liveness_stream.yml" + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/display_kafka_broker_info.yml" + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' + + when: kafka_broker is defined and kafka_broker != '' + +- block: + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_launch_stream_derive_leader_broker.yml" + when: kafka_stream is defined and kafka_stream == 'enabled' + + - set_fact: + kafka_broker_util: "/utils/apps/kafka/kafka_select_broker.yml" + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' + + when: kafka_broker is not defined or kafka_broker == '' + + + diff --git a/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-k8s-job.yml b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-k8s-job.yml new file mode 100644 index 00000000000..4598b03ed13 --- /dev/null +++ b/experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-k8s-job.yml @@ -0,0 +1,102 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + generateName: kafka-broker-network-loss- +spec: + template: + metadata: + labels: + experiment: kafka-broker-network-loss + spec: + # Placeholder that is updated by the executor for automated runs + # Provide appropriate SA (with desired permissions) if executed manually + serviceAccountName: %CHAOS_SERVICE_ACCOUNT% + restartPolicy: Never + containers: + - name: ansibletest + image: litmuschaos/ansible-runner:ci + imagePullPolicy: Always + env: + - name: ANSIBLE_STDOUT_CALLBACK + value: 'default' + + - name: KAFKA_KIND + value: 'statefulset' + + - name: KAFKA_LIVENESS_STREAM + value: 'enabled' + + - name: KAFKA_LIVENESS_IMAGE + value: 'litmuschaos/kafka-client:ci' + + # in milliseconds + - name: KAFKA_CONSUMER_TIMEOUT + value: '30000' + + # in milliseconds + - name: TOTAL_CHAOS_DURATION + value: '60000' + + - name: TARGET_CONTAINER + value: 'k8skafka' + + # in percentage + - name: NETWORK_PACKET_LOSS_PERCENTAGE + value: '80' + + - name: NETWORK_INTERFACE + value: 'eth0' + + - name: KAFKA_INSTANCE_NAME + value: '' + + - name: KAFKA_NAMESPACE + value: '' + + - name: KAFKA_LABEL + value: '' + + - name: KAFKA_BROKER + value: '' + + - name: KAFKA_REPLICATION_FACTOR + value: '' + + - name: KAFKA_SERVICE + value: '' + + - name: KAFKA_PORT + value: '' + + - name: ZOOKEEPER_NAMESPACE + value: '' + + - name: ZOOKEEPER_LABEL + value: '' + + - name: ZOOKEEPER_SERVICE + value: '' + + - name: ZOOKEEPER_PORT + value: '' + + ## env var that describes the library used to execute the chaos + ## supported values: pumba + - name: LIB + value: 'pumba' + + - name: LIB_IMAGE + value: 'gaiaadm/pumba:0.6.5' + + - name: CHAOSENGINE + value: '' + + - name: CHAOS_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + + command: ["/bin/bash"] + args: ["-c", "ansible-playbook ./experiments/kafka/kafka-broker-network-loss/kafka-broker-network-loss-ansible-logic.yml -vv -i /etc/ansible/hosts; exit 0"] + diff --git a/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-ansible-prerequisites.yml b/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-ansible-prerequisites.yml index 10293ba0a95..9cce94f7a0a 100644 --- a/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-ansible-prerequisites.yml +++ b/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-ansible-prerequisites.yml @@ -7,11 +7,11 @@ - set_fact: kafka_broker_util: "/utils/apps/kafka/kafka_liveness_stream.yml" - when: kafka_stream is defined and kafka_stream != '' + when: kafka_stream is defined and kafka_stream == 'enabled' - set_fact: kafka_broker_util: "/utils/apps/kafka/display_kafka_broker_info.yml" - when: kafka_stream is not defined or kafka_stream == '' + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' when: kafka_broker is defined and kafka_broker != '' @@ -19,11 +19,11 @@ - set_fact: kafka_broker_util: "/utils/apps/kafka/kafka_launch_stream_derive_leader_broker.yml" - when: kafka_stream is defined and kafka_stream != '' + when: kafka_stream is defined and kafka_stream == 'enabled' - set_fact: kafka_broker_util: "/utils/apps/kafka/kafka_select_broker.yml" - when: kafka_stream is not defined or kafka_stream == '' + when: kafka_stream is not defined or kafka_stream == '' or kafka_stream == 'disabled' when: kafka_broker is not defined or kafka_broker == '' diff --git a/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-k8s-job.yml b/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-k8s-job.yml index 9043f653ade..775c8ff8772 100644 --- a/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-k8s-job.yml +++ b/experiments/kafka/kafka-broker-pod-failure/kafka-broker-pod-failure-k8s-job.yml @@ -81,7 +81,7 @@ spec: value: '' ## env var that describes the library used to execute the chaos - ## default: litmus. Supported values: litmus, powerfulseal, chaoskube + ## default: litmus. Supported values: litmus, powerfulseal - name: LIB value: '' diff --git a/utils/apps/kafka/kafka_liveness_stream.yml b/utils/apps/kafka/kafka_liveness_stream.yml index 33216cf85af..1dfb3c036e9 100644 --- a/utils/apps/kafka/kafka_liveness_stream.yml +++ b/utils/apps/kafka/kafka_liveness_stream.yml @@ -20,7 +20,7 @@ dest: kafka_liveness.yml when: kafka_sasl_auth is defined and kafka_sasl_auth == 'enabled' -- name: Apply the pub-sub kafka liveness applicaton +- name: Apply the pub-sub kafka liveness application shell: kubectl apply -f kafka_liveness.yml -n {{ kafka_ns }} args: