diff --git a/kube/kubernetes_api_service.go b/kube/kubernetes_api_service.go index 4d6d27b..207f57b 100644 --- a/kube/kubernetes_api_service.go +++ b/kube/kubernetes_api_service.go @@ -18,12 +18,24 @@ import ( "k8s.io/client-go/rest" ) +type PrivilegedPodConfig struct { + NodeName string + ContainerName string + Image string + SocketPath string + Timeout time.Duration +} + +func NewCreatePodConfig() *PrivilegedPodConfig { + return &PrivilegedPodConfig{Timeout: 10 * time.Minute} +} + type KubernetesApiService interface { ExecuteCommand(podName string, containerName string, command []string, stdOut io.Writer) (int, error) DeletePod(podName string) error - CreatePrivilegedPod(nodeName string, containerName string, image string, socketPath string, timeout time.Duration) (*corev1.Pod, error) + CreatePrivilegedPod(config *PrivilegedPodConfig) (*corev1.Pod, error) UploadFile(localPath string, remotePath string, podName string, containerName string) error } @@ -104,16 +116,19 @@ func (k *KubernetesApiServiceImpl) DeletePod(podName string) error { return err } -func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containerName string, image string, socketPath string, timeout time.Duration) (*corev1.Pod, error) { - log.Debugf("creating privileged pod on remote node") +func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(config *PrivilegedPodConfig) (*corev1.Pod, error) { + log.Info("creating privileged pod on remote node") + log.Debugf("creating privileged pod with the following options: { %v }", config) - isSupported, err := k.IsSupportedContainerRuntime(nodeName) + hostNetwork := true + + isSupported, err := k.IsSupportedContainerRuntime(config.NodeName) if err != nil { return nil, err } if !isSupported { - return nil, errors.Errorf("Container runtime on node %s isn't supported. Supported container runtimes are: %v", nodeName, runtime.SupportedContainerRuntimes) + return nil, errors.Errorf("Container runtime on node %s isn't supported. Supported container runtimes are: %v", config.NodeName, runtime.SupportedContainerRuntimes) } typeMetadata := v1.TypeMeta{ @@ -129,12 +144,11 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe }, } + // Add Storage / Mounts + hostPathType := corev1.HostPathSocket + directoryType := corev1.HostPathDirectory + volumeMounts := []corev1.VolumeMount{ - { - Name: "container-socket", - ReadOnly: true, - MountPath: socketPath, - }, { Name: "host", ReadOnly: false, @@ -142,10 +156,40 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe }, } + volumes := []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: &directoryType, + }, + }, + }, + } + + if config.SocketPath != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "container-socket", + ReadOnly: true, + MountPath: config.SocketPath, + }) + volumes = append(volumes, corev1.Volume{ + Name: "container-socket", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: config.SocketPath, + Type: &hostPathType, + }, + }, + }) + } + + // Create Privileged container privileged := true privilegedContainer := corev1.Container{ - Name: containerName, - Image: image, + Name: config.ContainerName, + Image: config.Image, SecurityContext: &corev1.SecurityContext{ Privileged: &privileged, @@ -155,34 +199,13 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe VolumeMounts: volumeMounts, } - hostPathType := corev1.HostPathSocket - directoryType := corev1.HostPathDirectory - podSpecs := corev1.PodSpec{ - NodeName: nodeName, + NodeName: config.NodeName, RestartPolicy: corev1.RestartPolicyNever, HostPID: true, Containers: []corev1.Container{privilegedContainer}, - Volumes: []corev1.Volume{ - { - Name: "host", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/", - Type: &directoryType, - }, - }, - }, - { - Name: "container-socket", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: socketPath, - Type: &hostPathType, - }, - }, - }, - }, + Volumes: volumes, + HostNetwork: hostNetwork, } pod := corev1.Pod{ @@ -214,8 +237,8 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe log.Info("waiting for pod successful startup") - if !utils.RunWhileFalse(verifyPodState, timeout, 1*time.Second) { - return nil, errors.Errorf("failed to create pod within timeout (%s)", timeout) + if !utils.RunWhileFalse(verifyPodState, config.Timeout, 1*time.Second) { + return nil, errors.Errorf("failed to create pod within timeout (%s)", config.Timeout) } return createdPod, nil diff --git a/pkg/cmd/sniff.go b/pkg/cmd/sniff.go index 84aae65..4cd1ccf 100644 --- a/pkg/cmd/sniff.go +++ b/pkg/cmd/sniff.go @@ -18,16 +18,15 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" - _ "k8s.io/client-go/plugin/pkg/client/auth/azure" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" ) var ( @@ -172,7 +171,7 @@ func (o *Ksniff) Complete(cmd *cobra.Command, args []string) error { o.settings.UseDefaultImage = !cmd.Flag("image").Changed o.settings.UseDefaultTCPDumpImage = !cmd.Flag("tcpdump-image").Changed o.settings.UseDefaultSocketPath = !cmd.Flag("socket").Changed - + var err error if o.settings.UserSpecifiedVerboseMode { @@ -267,7 +266,7 @@ func (o *Ksniff) Validate() error { log.Infof("using tcpdump path at: '%s'", o.settings.UserSpecifiedLocalTcpdumpPath) } - pod, err := o.clientset.CoreV1().Pods(o.resultingContext.Namespace).Get(context.TODO(), o.settings.UserSpecifiedPodName, v1.GetOptions{}) + pod, err := o.clientset.CoreV1().Pods(o.resultingContext.Namespace).Get(context.TODO(), o.settings.UserSpecifiedPodName, metav1.GetOptions{}) if err != nil { return err } @@ -312,7 +311,9 @@ func (o *Ksniff) getPodSnifferService(pod *corev1.Pod) (sniffer.SnifferService, if o.settings.UserSpecifiedPrivilegedMode { log.Info("sniffing method: privileged pod") snifferVar, err = sniffer.NewPrivilegedPodRemoteSniffingService(o.settings, pod, kubernetesApiService) - + } else if o.settings.UserSpecifiedNodeMode { + log.Info("sniffing method: node") + snifferVar = sniffer.NewNodeSnifferService(o.settings, kubernetesApiService) } else { log.Info("sniffing method: upload static tcpdump") snifferVar = sniffer.NewUploadTcpdumpRemoteSniffingService(o.settings, kubernetesApiService) diff --git a/pkg/config/settings.go b/pkg/config/settings.go index 4e7f813..943511b 100644 --- a/pkg/config/settings.go +++ b/pkg/config/settings.go @@ -18,6 +18,8 @@ type KsniffSettings struct { UserSpecifiedRemoteTcpdumpPath string UserSpecifiedVerboseMode bool UserSpecifiedPrivilegedMode bool + UserSpecifiedNodeMode bool + UserSpecifiedNodeName string UserSpecifiedImage string DetectedPodNodeName string DetectedContainerId string @@ -52,6 +54,14 @@ type StaticTCPSnifferServiceConfig struct { UserSpecifiedFilter string } +type NodeSnifferServiceConfig struct { + Image string + UserSpecifiedInterface string + UserSpecifiedFilter string + NodeName string + UserSpecifiedPodCreateTimeout time.Duration +} + func NewKsniffSettings(streams genericclioptions.IOStreams) *KsniffSettings { return &KsniffSettings{} } diff --git a/pkg/service/sniffer/node_sniffer_service.go b/pkg/service/sniffer/node_sniffer_service.go new file mode 100644 index 0000000..08becd7 --- /dev/null +++ b/pkg/service/sniffer/node_sniffer_service.go @@ -0,0 +1,104 @@ +package sniffer + +import ( + "io" + + "ksniff/kube" + "ksniff/pkg/config" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" +) + +var defaultInterface = "any" + +type NodeSnifferService struct { + *config.NodeSnifferServiceConfig + privilegedPod *v1.Pod + privilegedContainerName string + targetInterface string + // TODO Replace Node Name + nodeName string + kubernetesApiService kube.KubernetesApiService +} + +func NewNodeSnifferService(options *config.KsniffSettings, service kube.KubernetesApiService) SnifferService { + nodeSnifferService := &NodeSnifferService{ + NodeSnifferServiceConfig: &config.NodeSnifferServiceConfig{ + Image: options.UserSpecifiedImage, + UserSpecifiedInterface: options.UserSpecifiedInterface, + UserSpecifiedFilter: options.UserSpecifiedFilter, + NodeName: options.UserSpecifiedNodeName, + UserSpecifiedPodCreateTimeout: options.UserSpecifiedPodCreateTimeout, + }, + privilegedContainerName: "node-sniff", + kubernetesApiService: service, + nodeName: options.DetectedPodNodeName, + targetInterface: defaultInterface, + } + + if options.UseDefaultImage { + nodeSnifferService.Image = "maintained/tcpdump" + } + + return nodeSnifferService +} + +func (nss *NodeSnifferService) Setup() error { + var err error + // TODO Create a Nodesniffer Object + log.Infof("creating privileged pod on node: '%s'", nss.nodeName) + log.Debugf("initiating sniff on node with option: '%v'", nss) + + podConfig := kube.PrivilegedPodConfig{ + // TODO Replace DetectedPodNodeName with PodName + NodeName: nss.nodeName, + ContainerName: nss.privilegedContainerName, + Image: nss.Image, + Timeout: nss.UserSpecifiedPodCreateTimeout, + } + + nss.privilegedPod, err = nss.kubernetesApiService.CreatePrivilegedPod(&podConfig) + if err != nil { + log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", nss.nodeName) + return err + } + + log.Infof("pod: '%s' created successfully on node: '%s'", nss.privilegedPod.Name, nss.nodeName) + + return nil +} + +func (nss *NodeSnifferService) Cleanup() error { + log.Infof("removing pod: '%s'", nss.privilegedPod.Name) + + err := nss.kubernetesApiService.DeletePod(nss.privilegedPod.Name) + if err != nil { + log.WithError(err).Errorf("failed to remove pod: '%s", nss.privilegedPod.Name) + return err + } + + log.Infof("pod: '%s' removed successfully", nss.privilegedPod.Name) + + return nil +} + +func buildTcpdumpCommand(netInterface string, filter string, tcpdumpImage string) []string { + return []string{"tcpdump", "-i", netInterface, "-U", "-w", "-", filter} +} + +func (nss *NodeSnifferService) Start(stdOut io.Writer) error { + log.Info("starting remote sniffing using privileged pod") + + command := buildTcpdumpCommand(nss.targetInterface, nss.UserSpecifiedFilter, nss.Image) + + exitCode, err := nss.kubernetesApiService.ExecuteCommand(nss.privilegedPod.Name, nss.privilegedContainerName, command, stdOut) + if err != nil { + log.WithError(err).Errorf("failed to start sniffing using privileged pod, exit code: '%d'", exitCode) + return err + } + + log.Info("remote sniffing using privileged pod completed") + + return nil +} diff --git a/pkg/service/sniffer/privileged_pod_sniffer_service.go b/pkg/service/sniffer/privileged_pod_sniffer_service.go index 7612cc7..cfe4c44 100644 --- a/pkg/service/sniffer/privileged_pod_sniffer_service.go +++ b/pkg/service/sniffer/privileged_pod_sniffer_service.go @@ -72,13 +72,17 @@ func (p *PrivilegedPodSnifferService) Setup() error { var err error log.Infof("creating privileged pod on node: '%s'", p.DetectedPodNodeName) - p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod( - p.DetectedPodNodeName, - p.privilegedContainerName, - p.Image, - p.SocketPath, - p.UserSpecifiedPodCreateTimeout, - ) + + podConfig := kube.PrivilegedPodConfig{ + NodeName: p.DetectedPodNodeName, + ContainerName: p.privilegedContainerName, + Image: p.Image, + SocketPath: p.SocketPath, + Timeout: p.UserSpecifiedPodCreateTimeout, + } + + p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod(&podConfig) + if err != nil { log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", p.DetectedPodNodeName) return err diff --git a/pkg/service/sniffer/privileged_pod_sniffer_service.go_backup b/pkg/service/sniffer/privileged_pod_sniffer_service.go_backup new file mode 100644 index 0000000..cfe4c44 --- /dev/null +++ b/pkg/service/sniffer/privileged_pod_sniffer_service.go_backup @@ -0,0 +1,178 @@ +package sniffer + +import ( + "bytes" + "fmt" + "io" + "strings" + + "ksniff/kube" + "ksniff/pkg/config" + "ksniff/pkg/service/sniffer/runtime" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" +) + +type PrivilegedPodSnifferService struct { + *config.PrivilegedSnifferServiceConfig + privilegedPod *corev1.Pod + privilegedContainerName string + targetProcessId *string + kubernetesApiService kube.KubernetesApiService + runtimeBridge runtime.ContainerRuntimeBridge +} + + +func NewPrivilegedPodRemoteSniffingService(knsiffSettings *config.KsniffSettings, pod *corev1.Pod, service kube.KubernetesApiService) (SnifferService, error) { + runtimeStr, containerID, err := getPodContainerRuntimeDetails(pod) + + if err != nil { + return nil, err + } + + bridge := runtime.NewContainerRuntimeBridge(runtimeStr) + + snifferService := &PrivilegedPodSnifferService{ + privilegedContainerName: "ksniff-privileged", + kubernetesApiService: service, + runtimeBridge: bridge, + PrivilegedSnifferServiceConfig: &config.PrivilegedSnifferServiceConfig{ + DetectedContainerId: containerID, + DetectedContainerRuntime: runtimeStr, + Image: knsiffSettings.Image, + TCPDumpImage: knsiffSettings.TCPDumpImage, + SocketPath: knsiffSettings.SocketPath, + DetectedPodNodeName: knsiffSettings.DetectedPodNodeName, + UserSpecifiedInterface: knsiffSettings.UserSpecifiedInterface, + UserSpecifiedFilter: knsiffSettings.UserSpecifiedFilter, + UserSpecifiedPodCreateTimeout: knsiffSettings.UserSpecifiedPodCreateTimeout, + }, + } + // Overwrite with defaults if not specified + if knsiffSettings.UseDefaultImage { + snifferService.Image = snifferService.runtimeBridge.GetDefaultImage() + } + + if knsiffSettings.UseDefaultTCPDumpImage { + snifferService.TCPDumpImage = snifferService.runtimeBridge.GetDefaultTCPImage() + } + + if knsiffSettings.UseDefaultSocketPath { + snifferService.SocketPath = snifferService.runtimeBridge.GetDefaultSocketPath() + } + + + return snifferService, nil + +} + +func (p *PrivilegedPodSnifferService) Setup() error { + var err error + + log.Infof("creating privileged pod on node: '%s'", p.DetectedPodNodeName) + + podConfig := kube.PrivilegedPodConfig{ + NodeName: p.DetectedPodNodeName, + ContainerName: p.privilegedContainerName, + Image: p.Image, + SocketPath: p.SocketPath, + Timeout: p.UserSpecifiedPodCreateTimeout, + } + + p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod(&podConfig) + + if err != nil { + log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", p.DetectedPodNodeName) + return err + } + + log.Infof("pod: '%s' created successfully on node: '%s'", p.privilegedPod.Name, p.DetectedPodNodeName) + + if p.runtimeBridge.NeedsPid() { + var buff bytes.Buffer + command := p.runtimeBridge.BuildInspectCommand(p.DetectedContainerId) + exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedContainerName, command, &buff) + if err != nil { + log.WithError(err).Errorf("failed to start sniffing using privileged pod, exit code: '%d'", exitCode) + } + p.targetProcessId, err = p.runtimeBridge.ExtractPid(buff.String()) + if err != nil { + return err + } + } + + return nil +} + +func (p *PrivilegedPodSnifferService) Cleanup() error { + log.Infof("removing privileged container: '%s'", p.privilegedContainerName) + + command := p.runtimeBridge.BuildCleanupCommand() + + exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedContainerName, command, &kube.NopWriter{}) + if err != nil { + log.WithError(err).Errorf("failed to remove privileged container: '%s', exit code: '%d', "+ + "please manually remove it", p.privilegedContainerName, exitCode) + } else { + log.Infof("privileged container: '%s' removed successfully", p.privilegedContainerName) + } + + log.Infof("removing pod: '%s'", p.privilegedPod.Name) + + err = p.kubernetesApiService.DeletePod(p.privilegedPod.Name) + if err != nil { + log.WithError(err).Errorf("failed to remove pod: '%s", p.privilegedPod.Name) + return err + } + + log.Infof("pod: '%s' removed successfully", p.privilegedPod.Name) + + return nil +} + +func (p *PrivilegedPodSnifferService) Start(stdOut io.Writer) error { + log.Info("starting remote sniffing using privileged pod") + + command := p.runtimeBridge.BuildTcpdumpCommand( + &p.DetectedContainerId, + p.UserSpecifiedInterface, + p.UserSpecifiedFilter, + p.targetProcessId, + p.SocketPath, + p.TCPDumpImage, + ) + + exitCode, err := p.kubernetesApiService.ExecuteCommand(p.privilegedPod.Name, p.privilegedContainerName, command, stdOut) + if err != nil { + log.WithError(err).Errorf("failed to start sniffing using privileged pod, exit code: '%d'", exitCode) + return err + } + + log.Info("remote sniffing using privileged pod completed") + + return nil +} + +// Collect information about the container runtime that is running the Pod +func getPodContainerRuntimeDetails(pod *corev1.Pod) (containerRuntime string, containerID string, err error) { + if len(pod.Spec.Containers) < 1 { + return "", "", fmt.Errorf("the pod provided does not have any containers") + } + containerName := pod.Spec.Containers[0].Name + + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerName == containerStatus.Name { + result := strings.Split(containerStatus.ContainerID, "://") + if len(result) != 2 { + break + } + containerRuntime = result[0] + containerID = result[1] + return + } + } + err = errors.Errorf("couldn't find container: '%s' in pod: '%s'", containerName, pod.Name) + return +} \ No newline at end of file