support draining multiple node #469
+85
−48
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -49,16 +49,16 @@ func PrepareNodeDrain(experimentsDetails *experimentTypes.ExperimentDetails, cli | |
| common.WaitForDuration(experimentsDetails.RampTime) | ||
| } | ||
|
|
||
| if experimentsDetails.TargetNodes == "" { | ||
| //Select node for kubelet-service-kill | ||
| experimentsDetails.TargetNodes, err = common.GetNodeName(experimentsDetails.AppNS, experimentsDetails.AppLabel, experimentsDetails.NodeLabel, clients) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| if experimentsDetails.EngineName != "" { | ||
| msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on " + experimentsDetails.TargetNodes + " node" | ||
| types.SetEngineEventAttributes(eventsDetails, types.ChaosInject, msg, "Normal", chaosDetails) | ||
| events.GenerateEvents(eventsDetails, clients, chaosDetails, "ChaosEngine") | ||
| } | ||
|
|
@@ -114,45 +114,65 @@ func PrepareNodeDrain(experimentsDetails *experimentTypes.ExperimentDetails, cli | |
| // drainNode drain the application node | ||
| func drainNode(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails) error { | ||
|
|
||
| targetNodes := strings.Split(experimentsDetails.TargetNodes, ",") | ||
| if len(targetNodes) == 0 { | ||
| return errors.Errorf("No target nodes provided, expected the comma-separated names of one or more nodes") | ||
| } | ||
|
|
||
| log.Infof("Target nodes list: %v", targetNodes) | ||
| for _, targetNode := range targetNodes { | ||
|
There was a problem hiding this comment. Can we also handle the panic case when the length of the list is zero and give an error message to provide the target node name. There was a problem hiding this comment. When we run chaos for more nodes we could perform the pre and post chaos check for all the target nodes. There was a problem hiding this comment. We can also change |
||
|
|
||
| select { | ||
| case <-inject: | ||
| // stopping the chaos execution, if abort signal received | ||
| os.Exit(0) | ||
| default: | ||
| log.Infof("[Inject]: Draining the %v node", targetNode) | ||
|
|
||
| command := exec.Command("kubectl", "drain", targetNode, "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--timeout", strconv.Itoa(experimentsDetails.ChaosDuration)+"s") | ||
| var out, stderr bytes.Buffer | ||
| command.Stdout = &out | ||
| command.Stderr = &stderr | ||
| if err := command.Run(); err != nil { | ||
| log.Infof("Error String: %v", stderr.String()) | ||
| return errors.Errorf("Unable to drain the %v node, err: %v", targetNode, err) | ||
|
There was a problem hiding this comment. We can pass the stderr here since the err will contain the exit code description e.g. |
||
| } | ||
|
|
||
| common.SetTargets(targetNode, "injected", "node", chaosDetails) | ||
|
|
||
| err = retry. | ||
| Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)). | ||
| Wait(time.Duration(experimentsDetails.Delay) * time.Second). | ||
| Try(func(attempt uint) error { | ||
| nodeSpec, err := clients.KubeClient.CoreV1().Nodes().Get(targetNode, v1.GetOptions{}) | ||
| if err != nil { | ||
| if apierrors.IsNotFound(err) { | ||
| return nil | ||
|
There was a problem hiding this comment. Shall we add a log here to specify that the resource was not found? |
||
| } else { | ||
| return err | ||
| } | ||
| } | ||
| if !nodeSpec.Spec.Unschedulable { | ||
| return errors.Errorf("%v node is not in unschedulable state", targetNode) | ||
| } | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // uncordonNode uncordon the application node | ||
| func uncordonNode(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails) error { | ||
|
|
||
| targetNodes := strings.Split(experimentsDetails.TargetNodes, ",") | ||
| if len(targetNodes) == 0 { | ||
| return errors.Errorf("No target nodes provided, expected the comma-separated names of one or more nodes") | ||
| } | ||
|
|
||
| for _, targetNode := range targetNodes { | ||
|
|
||
| //Check node exist before uncordon the node | ||
|
|
@@ -183,7 +203,11 @@ func uncordonNode(experimentsDetails *experimentTypes.ExperimentDetails, clients | |
| Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)). | ||
| Wait(time.Duration(experimentsDetails.Delay) * time.Second). | ||
| Try(func(attempt uint) error { | ||
| targetNodes := strings.Split(experimentsDetails.TargetNodes, ",") | ||
| if len(targetNodes) == 0 { | ||
|
There was a problem hiding this comment. Same comment as above, this check will fail for an empty string. |
||
| return errors.Errorf("No target nodes provided, expected the comma-separated names of one or more nodes") | ||
| } | ||
|
|
||
| for _, targetNode := range targetNodes { | ||
| nodeSpec, err := clients.KubeClient.CoreV1().Nodes().Get(targetNode, v1.GetOptions{}) | ||
| if err != nil { | ||
|
|
@@ -194,7 +218,7 @@ func uncordonNode(experimentsDetails *experimentTypes.ExperimentDetails, clients | |
| } | ||
| } | ||
| if nodeSpec.Spec.Unschedulable { | ||
| return errors.Errorf("%v node is in unschedulable state", targetNode) | ||
| } | ||
| } | ||
| return nil | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Even for an empty string value in
experimentsDetails.TargetNodes, thelen(targetNodes)will be equal to 1. We can modify the check as follows: