Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use "ip netns exec" instead of "nsenter" to handle pod restart in network-chaos #702

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+52 −12
Open

Use "ip netns exec" instead of "nsenter" to handle pod restart in network-chaos #702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cb08286
Adding GetNetNs function to retrieve the netns ID.
Calvinaud May 24, 2024
e570b79
Running gofmt to update netem.go
Calvinaud May 24, 2024
2bfd118
Remove unused source variable in GetNS function.
Calvinaud May 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ COPY --from=dep /usr/bin/sudo /usr/bin/sudo
COPY --from=dep /usr/lib/sudo /usr/lib/sudo
COPY --from=dep /sbin/tc /sbin/
COPY --from=dep /sbin/iptables /sbin/
COPY --from=dep /sbin/ip /sbin/

# Copying Necessary Files
COPY ./pkg/cloud/aws/common/ssm-docs/LitmusChaos-AWS-SSM-Docs.yml .
51 changes: 39 additions & 12 deletions chaoslib/litmus/network-chaos/helper/netem.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package helper

import (
"bytes"
"fmt"
"github.com/litmuschaos/litmus-go/pkg/cerrors"
"github.com/litmuschaos/litmus-go/pkg/events"
Expand Down Expand Up @@ -76,6 +77,26 @@ func Helper(clients clients.ClientSets) {

}

// Retrieve ID of the network namespace of the target container
func GetNetNS(pid int) (string, error) {

cmd := exec.Command("sudo", "ip", "netns", "identify", fmt.Sprintf("%d", pid))

var out, stdErr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stdErr
if err := cmd.Run(); err != nil {
return "", err
}

netns := out.String()
// Removing trailing newline character
netnstrim := netns[:len(netns)-1]
log.Infof("[Info]: Process PID=%d has netns ID=%s", pid, netnstrim)

return netnstrim, nil
}

// preparePodNetworkChaos contains the prepration steps before chaos injection
func preparePodNetworkChaos(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails) error {

Expand Down Expand Up @@ -110,6 +131,11 @@ func preparePodNetworkChaos(experimentsDetails *experimentTypes.ExperimentDetail
return stacktrace.Propagate(err, "could not get container pid")
}

td.NetNS, err = GetNetNS(td.Pid)
if err != nil {
return stacktrace.Propagate(err, "could not get netns id")
}

targets = append(targets, td)
}

Expand Down Expand Up @@ -171,14 +197,14 @@ func preparePodNetworkChaos(experimentsDetails *experimentTypes.ExperimentDetail
}

// injectChaos inject the network chaos in target container
// it is using nsenter command to enter into network namespace of target container
// it is using ip netns command to enter into network namespace of target container
// and execute the netem command inside it.
func injectChaos(netInterface string, target targetDetails) error {

netemCommands := os.Getenv("NETEM_COMMAND")

if len(target.DestinationIps) == 0 && len(sPorts) == 0 && len(dPorts) == 0 && len(whitelistDPorts) == 0 && len(whitelistSPorts) == 0 {
tc := fmt.Sprintf("sudo nsenter -t %d -n tc qdisc replace dev %s root netem %v", target.Pid, netInterface, netemCommands)
tc := fmt.Sprintf("sudo ip netns exec %s tc qdisc replace dev %s root netem %v", target.NetNS, netInterface, netemCommands)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create tc rules", target.Source); err != nil {
return err
Expand All @@ -187,15 +213,15 @@ func injectChaos(netInterface string, target targetDetails) error {

// Create a priority-based queue
// This instantly creates classes 1:1, 1:2, 1:3
priority := fmt.Sprintf("sudo nsenter -t %v -n tc qdisc replace dev %v root handle 1: prio", target.Pid, netInterface)
priority := fmt.Sprintf("sudo ip netns exec %s tc qdisc replace dev %v root handle 1: prio", target.NetNS, netInterface)
log.Info(priority)
if err := common.RunBashCommand(priority, "failed to create priority-based queue", target.Source); err != nil {
return err
}

// Add queueing discipline for 1:3 class.
// No traffic is going through 1:3 yet
traffic := fmt.Sprintf("sudo nsenter -t %v -n tc qdisc replace dev %v parent 1:3 netem %v", target.Pid, netInterface, netemCommands)
traffic := fmt.Sprintf("sudo ip netns exec %s tc qdisc replace dev %v parent 1:3 netem %v", target.NetNS, netInterface, netemCommands)
log.Info(traffic)
if err := common.RunBashCommand(traffic, "failed to create netem queueing discipline", target.Source); err != nil {
return err
Expand All @@ -204,7 +230,7 @@ func injectChaos(netInterface string, target targetDetails) error {
if len(whitelistDPorts) != 0 || len(whitelistSPorts) != 0 {
for _, port := range whitelistDPorts {
//redirect traffic to specific dport through band 2
tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 2 u32 match ip dport %v 0xffff flowid 1:2", target.Pid, netInterface, port)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 2 u32 match ip dport %v 0xffff flowid 1:2", target.NetNS, netInterface, port)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create whitelist dport match filters", target.Source); err != nil {
return err
Expand All @@ -213,14 +239,14 @@ func injectChaos(netInterface string, target targetDetails) error {

for _, port := range whitelistSPorts {
//redirect traffic to specific sport through band 2
tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 2 u32 match ip sport %v 0xffff flowid 1:2", target.Pid, netInterface, port)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 2 u32 match ip sport %v 0xffff flowid 1:2", target.NetNS, netInterface, port)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create whitelist sport match filters", target.Source); err != nil {
return err
}
}

tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dst 0.0.0.0/0 flowid 1:3", target.Pid, netInterface)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dst 0.0.0.0/0 flowid 1:3", target.NetNS, netInterface)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create rule for all ports match filters", target.Source); err != nil {
return err
Expand All @@ -229,9 +255,9 @@ func injectChaos(netInterface string, target targetDetails) error {

for _, ip := range target.DestinationIps {
// redirect traffic to specific IP through band 3
tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dst %v flowid 1:3", target.Pid, netInterface, ip)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dst %v flowid 1:3", target.NetNS, netInterface, ip)
if strings.Contains(ip, ":") {
tc = fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip6 dst %v flowid 1:3", target.Pid, netInterface, ip)
tc = fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip6 dst %v flowid 1:3", target.NetNS, netInterface, ip)
}
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create destination ips match filters", target.Source); err != nil {
Expand All @@ -241,7 +267,7 @@ func injectChaos(netInterface string, target targetDetails) error {

for _, port := range sPorts {
//redirect traffic to specific sport through band 3
tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip sport %v 0xffff flowid 1:3", target.Pid, netInterface, port)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip sport %v 0xffff flowid 1:3", target.NetNS, netInterface, port)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create source ports match filters", target.Source); err != nil {
return err
Expand All @@ -250,7 +276,7 @@ func injectChaos(netInterface string, target targetDetails) error {

for _, port := range dPorts {
//redirect traffic to specific dport through band 3
tc := fmt.Sprintf("sudo nsenter -t %v -n tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dport %v 0xffff flowid 1:3", target.Pid, netInterface, port)
tc := fmt.Sprintf("sudo ip netns exec %s tc filter add dev %v protocol ip parent 1:0 prio 3 u32 match ip dport %v 0xffff flowid 1:3", target.NetNS, netInterface, port)
log.Info(tc)
if err := common.RunBashCommand(tc, "failed to create destination ports match filters", target.Source); err != nil {
return err
Expand All @@ -266,7 +292,7 @@ func injectChaos(netInterface string, target targetDetails) error {
// killnetem kill the netem process for all the target containers
func killnetem(target targetDetails, networkInterface string) (bool, error) {

tc := fmt.Sprintf("sudo nsenter -t %d -n tc qdisc delete dev %s root", target.Pid, networkInterface)
tc := fmt.Sprintf("sudo ip netns exec %s tc qdisc delete dev %s root", target.NetNS, networkInterface)
cmd := exec.Command("/bin/bash", "-c", tc)
out, err := cmd.CombinedOutput()

Expand All @@ -292,6 +318,7 @@ type targetDetails struct {
TargetContainer string
ContainerId string
Pid int
NetNS string
Source string
}

Expand Down
12 changes: 12 additions & 0 deletions chaoslib/litmus/network-chaos/lib/network-chaos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,14 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
},
},
},
{
Name: "netns",
VolumeSource: apiv1.VolumeSource{
HostPath: &apiv1.HostPathVolumeSource{
Path: "/var/run/netns",
},
},
},
},

Containers: []apiv1.Container{
Expand All @@ -245,6 +253,10 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Name: "cri-socket",
MountPath: experimentsDetails.SocketPath,
},
{
Name: "netns",
MountPath: "/var/run/netns",
},
},
SecurityContext: &apiv1.SecurityContext{
Privileged: &privilegedEnable,
Expand Down
Loading