Skip to content

Commit

Permalink
node-drain: run pre/post checks on all target nodes
Browse files Browse the repository at this point in the history
  • Loading branch information
calebxu-hcl committed Jan 17, 2022
1 parent 248daf9 commit bfa3c9b
Showing 1 changed file with 24 additions and 11 deletions.
35 changes: 24 additions & 11 deletions experiments/generic/node-drain/experiment/node-drain.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package experiment

import (
"os"
"strings"

"github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1"
litmusLIB "github.com/litmuschaos/litmus-go/chaoslib/litmus/node-drain/lib"
Expand Down Expand Up @@ -68,6 +69,14 @@ func NodeDrain(clients clients.ClientSets) {
"Chaos Duration": experimentsDetails.ChaosDuration,
})

targetNodes := strings.Split(experimentsDetails.TargetNodes, ",")
if len(targetNodes) == 0 {
log.Errorf("No target nodes provided, expected the comma-separated names of one or more nodes")
failStep := "[pre-chaos]: No target nodes provided, expected the comma-separated names of one or more nodes"
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}

// Calling AbortWatcher go routine, it will continuously watch for the abort signal and generate the required events and result
go common.AbortWatcherWithoutExit(experimentsDetails.ExperimentName, clients, &resultDetails, &chaosDetails, &eventsDetails)

Expand All @@ -93,13 +102,15 @@ func NodeDrain(clients clients.ClientSets) {

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err := status.CheckNodeStatus(experimentsDetails.TargetNodes, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Target nodes are not in the ready state, err: %v", err)
failStep := "[pre-chaos]: Failed to verify the status of nodes, err: " + err.Error()
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, "NUT: Not Ready", "Warning", &chaosDetails)
events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
for _, targetNode := range targetNodes {
if err := status.CheckNodeStatus(targetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Target nodes are not in the ready state, err: %v", err)
failStep := "[pre-chaos]: Failed to verify the status of nodes, err: " + err.Error()
types.SetEngineEventAttributes(&eventsDetails, types.PreChaosCheck, "NUT: Not Ready", "Warning", &chaosDetails)
events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
Expand Down Expand Up @@ -166,10 +177,12 @@ func NodeDrain(clients clients.ClientSets) {

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err := status.CheckNodeStatus(experimentsDetails.TargetNodes, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, "NUT: Not Ready", "Warning", &chaosDetails)
events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
for _, targetNode := range targetNodes {
if err := status.CheckNodeStatus(targetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
types.SetEngineEventAttributes(&eventsDetails, types.PostChaosCheck, "NUT: Not Ready", "Warning", &chaosDetails)
events.GenerateEvents(&eventsDetails, clients, &chaosDetails, "ChaosEngine")
}
}

if experimentsDetails.EngineName != "" {
Expand Down

0 comments on commit bfa3c9b

Please sign in to comment.