From 044046a14d969c6c40ed15387bd3f954fe413b50 Mon Sep 17 00:00:00 2001 From: Kirtana Ashok Date: Thu, 5 Dec 2024 16:15:31 -0800 Subject: [PATCH] Test gcs-sidecar in uvm Signed-off-by: Kirtana Ashok --- cmd/gcs-sidecar/main.go | 266 +++++++++++++++++++++++++++++++++ internal/gcs/protocol.go | 25 ++++ internal/oci/uvm.go | 1 + internal/uvm/create.go | 4 + internal/uvm/create_wcow.go | 72 ++++++++- pkg/annotations/annotations.go | 4 + 6 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 cmd/gcs-sidecar/main.go diff --git a/cmd/gcs-sidecar/main.go b/cmd/gcs-sidecar/main.go new file mode 100644 index 0000000000..21d0e8a345 --- /dev/null +++ b/cmd/gcs-sidecar/main.go @@ -0,0 +1,266 @@ +package main + +import ( + "context" + "fmt" + "log" + "net" + "os" + "sync" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/gcs" + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/debug" +) + +const maxMsgSize = 0x10000 + +type handler struct { + fromsvc chan error +} + +// New guid for sidecar gcs service +// ae8da506-a019-4553-a52b-902bc0fa0411 +var WindowsSidecarGcsHvsockServiceID = guid.GUID{ + Data1: 0xae8da506, + Data2: 0xa019, + Data3: 0x4553, + Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, +} + +// Accepts new connection closes listener +func acceptAndClose(ctx context.Context, l net.Listener) (net.Conn, error) { + var conn net.Conn + ch := make(chan error) + go func() { + var err error + conn, err = l.Accept() + ch <- err + }() + select { + case err := <-ch: + l.Close() + return conn, err + case <-ctx.Done(): + } + + l.Close() + + err := <-ch + if err == nil { + return conn, err + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, err +} + +func recvFromShimAndForward(hcsshimCon *winio.HvsockConn, gcsCon net.Conn, wg *sync.WaitGroup) { + defer wg.Done() + + buffer := make([]byte, maxMsgSize) + for { + length, err := hcsshimCon.Read(buffer) + if err != nil { + fmt.Printf("Error reading from hcsshim: %v", err) + log.Printf("Error reading from hcsshim: %v", err) + return + } + + str := string(buffer[:length]) + fmt.Printf("Received len %v from shim: %s\n", length, str) + log.Printf("Received len %v from shim: %s\n", length, str) + + // TODO: Dereference and call policy enforcer as needed! + + // Forward message to inbox gcs + _, err = gcsCon.Write([]byte(str)) + if err != nil { + fmt.Printf("Error forwarding from sidecar to inbox: %v", err) + log.Printf("Error forwarding from sidecar to inbox: %v", err) + return + } + } +} + +func recvFromGcsAndForward(gcsCon /*readfrom */ net.Conn, hcsshimCon *winio.HvsockConn, wg *sync.WaitGroup) { + defer wg.Done() + + buffer := make([]byte, maxMsgSize) + for { + length, err := gcsCon.Read(buffer) + if err != nil { + fmt.Printf("Error reading from inbox gcs: %v", err) + log.Printf("Error reading from inbox gcs: %v", err) + return + } + + str := string(buffer[:length]) + fmt.Printf("Received len %v from InboxGCS: %s\n", length, str) + log.Printf("Received len %v from InboxGCS: %s\n", length, str) + + // TODO: Deferencing/unmounting/cleanup here on error before forwarding + // response to hcsshim + + _, err = hcsshimCon.Write([]byte(str)) + if err != nil { + fmt.Printf("Error forwarding from inbox to shim: %v", err) + log.Printf("Error forwarding from inbox to shim: %v", err) + return + } + } +} + +func startSendAndRecvLoops(shimCon *winio.HvsockConn, gcsCon net.Conn) { + var wg sync.WaitGroup + wg.Add(2) + defer wg.Wait() + + go recvFromShimAndForward(shimCon, gcsCon, &wg) + go recvFromGcsAndForward(gcsCon, shimCon, &wg) +} + +func (m *handler) Execute(args []string, r <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { + const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.AcceptPauseAndContinue + + status <- svc.Status{State: svc.StartPending, Accepts: 0} + // unblock runService() + m.fromsvc <- nil + + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + +loop: + for { + select { + case c := <-r: + switch c.Cmd { + case svc.Interrogate: + status <- c.CurrentStatus + case svc.Stop, svc.Shutdown: + log.Print("Shutting service...!") + // TODO: service stop?! + break loop + case svc.Pause: + status <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted} + case svc.Continue: + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + default: + log.Printf("Unexpected service control request #%d", c) + } + } + } + + status <- svc.Status{State: svc.StopPending} + return false, 1 +} + +func runService(name string, isDebug bool) error { + h := &handler{ + fromsvc: make(chan error), + } + + var err error + go func() { + if isDebug { + err := debug.Run(name, h) + if err != nil { + log.Fatalf("Error running service in debug mode.Err: %v", err) + } + } else { + err := svc.Run(name, h) + if err != nil { + log.Fatalf("Error running service in Service Control mode.Err %v", err) + } + } + h.fromsvc <- err + }() + + // Wait for the first signal from the service handler. + log.Printf("waiting for first signal from service handler\n") + err = <-h.fromsvc + if err != nil { + return err + } + return nil + +} + +func main() { + f, err := os.OpenFile("C:\\gcs-sidecar-logs.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + fmt.Printf("error opening file: %v", err) + log.Fatalf("error opening file: %v", err) + } + defer f.Close() + + log.SetOutput(f) + + type srvResp struct { + err error + } + + chsrv := make(chan error) + go func() { + defer close(chsrv) + + if err := runService("gcs-sidecar", false); err != nil { + log.Fatalf("error starting gcs-sidecar service: %v", err) + } + + chsrv <- err + }() + + select { + // case <-ctx.Done(): + // return ctx.Err() + case r := <-chsrv: + if r != nil { + log.Fatal(r) + } + } + + ctx := context.Background() + // 1. Setup connection with hcsshim external gcs connection + hvsockAddr := &winio.HvsockAddr{ + VMID: gcs.HV_GUID_PARENT, + ServiceID: gcs.WindowsSidecarGcsHvsockServiceID, + } + fmt.Printf("Dialing to hcsshim external bridge at address %v", hvsockAddr) + log.Printf("Dialing to hcsshim external bridge at address %v", hvsockAddr) + + shimCon, err := winio.Dial(ctx, hvsockAddr) + if err != nil { + fmt.Printf("Error dialing hcsshim external bridge at address %v", hvsockAddr) + log.Printf("Error dialing hcsshim external bridge at address %v", hvsockAddr) + return + } + + // 2. Start external server to connect with inbox GCS + listener, err := winio.ListenHvsock(&winio.HvsockAddr{ + VMID: gcs.HV_GUID_LOOPBACK, + //HV_GUID_PARENT, + ServiceID: gcs.WindowsGcsHvsockServiceID, + }) + if err != nil { + log.Printf("Error to start server for sidecar <-> inbox gcs communication: %v", err) + fmt.Printf("Error to start server for sidecar <-> inbox gcs communication: %v", err) + return + } + + var gcsListener net.Listener + gcsListener = listener + + gcsCon, err := acceptAndClose(ctx, gcsListener) + if err != nil { + fmt.Printf("Err accepting inbox GCS connection %v", err) + log.Printf("Err accepting inbox GCS connection %v", err) + return + } + + // 3. start the send and receive loops + startSendAndRecvLoops(shimCon, gcsCon) +} diff --git a/internal/gcs/protocol.go b/internal/gcs/protocol.go index 94e55e4c1e..531af3c248 100644 --- a/internal/gcs/protocol.go +++ b/internal/gcs/protocol.go @@ -12,6 +12,31 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) +// e0e16197-dd56-4a10-9195-5ee7a155a838 +var HV_GUID_LOOPBACK = guid.GUID{ + Data1: 0xe0e16197, + Data2: 0xdd56, + Data3: 0x4a10, + Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38}, +} + +// a42e7cda-d03f-480c-9cc2-a4de20abb878 +var HV_GUID_PARENT = guid.GUID{ + Data1: 0xa42e7cda, + Data2: 0xd03f, + Data3: 0x480c, + Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78}, +} + +// WindowsSidecarGcsHvsockServiceID is the hvsock service ID that the Windows GCS +// sidecar will connect to. +var WindowsSidecarGcsHvsockServiceID = guid.GUID{ + Data1: 0xae8da506, + Data2: 0xa019, + Data3: 0x4553, + Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, +} + // LinuxGcsVsockPort is the vsock port number that the Linux GCS will // connect to. const LinuxGcsVsockPort = 0x40000000 diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index caf698a645..6044ce2818 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -305,6 +305,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( wopts.NoDirectMap = ParseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) wopts.NoInheritHostTimezone = ParseAnnotationsBool(ctx, s.Annotations, annotations.NoInheritHostTimezone, wopts.NoInheritHostTimezone) wopts.AdditionalRegistryKeys = parseAdditionalRegistryValues(ctx, s.Annotations) + wopts.WcowSecurityPolicy = ParseAnnotationsString(s.Annotations, annotations.WcowSecurityPolicy, wopts.WcowSecurityPolicy) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) return wopts, nil } diff --git a/internal/uvm/create.go b/internal/uvm/create.go index 6a16e64be5..4cb637e8ac 100644 --- a/internal/uvm/create.go +++ b/internal/uvm/create.go @@ -97,6 +97,10 @@ type Options struct { // NoWritableFileShares disables adding any writable vSMB and Plan9 shares to the UVM NoWritableFileShares bool + // WcowSecurityPolicy is used to specify a security policy for cwcow to enforce + // in sidecar gcs. + WcowSecurityPolicy string + // The number of SCSI controllers. Defaults to 1 for WCOW and 4 for LCOW SCSIControllerCount uint32 diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 7c4107b1fe..363c6249c8 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -63,8 +63,10 @@ func (uvm *UtilityVM) startExternalGcsListener(ctx context.Context) error { log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge") l, err := winio.ListenHvsock(&winio.HvsockAddr{ - VMID: uvm.runtimeID, - ServiceID: gcs.WindowsGcsHvsockServiceID, + VMID: uvm.runtimeID, + // gcs.HV_GUID_PARENT, + ServiceID: gcs.WindowsSidecarGcsHvsockServiceID, + //gcs.WindowsGcsHvsockServiceID, }) if err != nil { return err @@ -152,6 +154,72 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv registryChanges.AddValues = append(registryChanges.AddValues, opts.AdditionalRegistryKeys...) + // Temporary hack to start up windows sidecar gcs in the uvm + isCWCOW := true + /* TODO: uncomment after POC + if opts.WcowSecurityPolicy != "" { + isCWCOW = true + } + */ + if isCWCOW { + registryChanges.AddValues = append(registryChanges.AddValues, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "DisplayName", + StringValue: "gcs-sidecar", + Type_: "String", + }, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "ErrorControl", + DWordValue: 1, + Type_: "DWord", + }, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "ImagePath", + StringValue: "C:\\Windows\\System32\\gcs-sidecar.exe", + Type_: "String", + }, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "ObjectName", + StringValue: "LocalSystem", + Type_: "String", + }, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "Start", + DWordValue: 2, + Type_: "DWord", + }, + hcsschema.RegistryValue{ + Key: &hcsschema.RegistryKey{ + Hive: "System", + Name: "CurrentControlSet\\Services\\gcs-sidecar", + }, + Name: "Type", + DWordValue: 16, + Type_: "DWord", + }, + ) + } + processor := &hcsschema.Processor2{ Count: uvm.processorCount, Limit: opts.ProcessorLimit, diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index ac0823cdf3..a7a8216143 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -221,6 +221,10 @@ const ( // SecurityPolicy is used to specify a security policy for opengcs to enforce. SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // WcowSecurityPolicy is used to specify a security policy for cwcow to enforce + // in sidecar gcs. + WcowSecurityPolicy = "io.microsoft.virtualmachine.wcow.securitypolicy" + // SecurityPolicyEnforcer is used to specify which enforcer to initialize (open-door, standard or rego). // This allows for better fallback mechanics. SecurityPolicyEnforcer = "io.microsoft.virtualmachine.lcow.enforcer"