diff --git a/go.mod b/go.mod index c9968ef544f..5b4830d65dc 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/avast/retry-go v3.0.0+incompatible github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/containerd v1.7.13 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index fcfaa4eae50..e331dbdff6d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index cfd5d42f858..48df35f4921 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -512,7 +512,7 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod for k, v := range config.Inputs { value := nestedMapLookup(ghc.Event, "inputs", k) if value == nil { - v.Default.Decode(&value) + _ = v.Default.Decode(&value) } if v.Type == "boolean" { inputs[k] = value == "true" diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 8abc4353cac..1303c47c2de 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -647,6 +647,9 @@ func (rc *RunContext) startContainer() common.Executor { if rc.IsHostEnv(ctx) { return rc.startHostEnvironment()(ctx) } + if rc.IsTartEnv(ctx) { + return rc.startTartEnvironment()(ctx) + } return rc.startJobContainer()(ctx) } } @@ -657,6 +660,12 @@ func (rc *RunContext) IsHostEnv(ctx context.Context) bool { return image == "" && strings.EqualFold(platform, "-self-hosted") } +func (rc *RunContext) IsTartEnv(ctx context.Context) bool { + platform := rc.runsOnImage(ctx) + image := rc.containerImage(ctx) + return image == "" && strings.HasPrefix(platform, "tart://") +} + func (rc *RunContext) stopContainer() common.Executor { return rc.stopJobContainer() } diff --git a/pkg/runner/run_context_darwin.go b/pkg/runner/run_context_darwin.go new file mode 100644 index 00000000000..6930b0f4505 --- /dev/null +++ b/pkg/runner/run_context_darwin.go @@ -0,0 +1,111 @@ +package runner + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" + "github.com/nektos/act/pkg/tart" +) + +func (rc *RunContext) startTartEnvironment() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0o777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0o777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0o777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + platImage := rc.runsOnImage(ctx) + platURI, _ := url.Parse(platImage) + query := platURI.Query() + tenv := &tart.Environment{ + HostEnvironment: container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + }, + Config: tart.Config{ + SSHUsername: "admin", + SSHPassword: "admin", + Softnet: query.Get("softnet") == "1", + Headless: query.Get("headless") != "0", + AlwaysPull: query.Get("pull") != "0", + }, + Env: &tart.Env{ + JobImage: platURI.Host + platURI.EscapedPath(), + JobID: rc.jobContainerName(), + }, + Miscpath: miscpath, + } + rc.JobContainer = tenv + if query.Has("sshusername") { + tenv.Config.SSHUsername = query.Get("sshusername") + } + if query.Has("sshpassword") { + tenv.Config.SSHPassword = query.Get("sshpassword") + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + for k, v := range rc.JobContainer.GetRunnerContext(ctx) { + if v, ok := v.(string); ok { + rc.Env[fmt.Sprintf("RUNNER_%s", strings.ToUpper(k))] = v + } + } + // for _, env := range os.Environ() { + // if k, v, ok := strings.Cut(env, "="); ok { + // // don't override + // if _, ok := rc.Env[k]; !ok { + // rc.Env[k] = v + // } + // } + // } + + return common.NewPipelineExecutor( + // rc.JobContainer.Remove(), + rc.JobContainer.Start(false), + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0o644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0o666, + Body: "", + }), + )(ctx) + } +} diff --git a/pkg/runner/run_context_other.go b/pkg/runner/run_context_other.go new file mode 100644 index 00000000000..3039b33986f --- /dev/null +++ b/pkg/runner/run_context_other.go @@ -0,0 +1,16 @@ +//go:build !darwin + +package runner + +import ( + "context" + "fmt" + + "github.com/nektos/act/pkg/common" +) + +func (rc *RunContext) startTartEnvironment() common.Executor { + return func(_ context.Context) error { + return fmt.Errorf("You need macOS for tart") + } +} diff --git a/pkg/tart/config_darwin.go b/pkg/tart/config_darwin.go new file mode 100644 index 00000000000..861f8d2756f --- /dev/null +++ b/pkg/tart/config_darwin.go @@ -0,0 +1,9 @@ +package tart + +type Config struct { + SSHUsername string + SSHPassword string + Softnet bool + Headless bool + AlwaysPull bool +} diff --git a/pkg/tart/env_darwin.go b/pkg/tart/env_darwin.go new file mode 100644 index 00000000000..d4a2c4ac3dd --- /dev/null +++ b/pkg/tart/env_darwin.go @@ -0,0 +1,18 @@ +package tart + +type Env struct { + JobID string + JobImage string + FailureExitCode int + Registry *Registry +} + +type Registry struct { + Address string + User string + Password string +} + +func (e Env) VirtualMachineID() string { + return e.JobID +} diff --git a/pkg/tart/environment_darwin.go b/pkg/tart/environment_darwin.go new file mode 100644 index 00000000000..30d97c0fd55 --- /dev/null +++ b/pkg/tart/environment_darwin.go @@ -0,0 +1,218 @@ +package tart + +import ( + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/kballard/go-shellquote" + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" +) + +type Environment struct { + container.HostEnvironment + vm *VM + Config Config + Env *Env + Miscpath string +} + +// "/Volumes/My Shared Files/act/" +func (e *Environment) ToHostPath(path string) string { + actPath := filepath.Clean("/private/tmp/act/") + altPath := filepath.Clean(path) + if strings.HasPrefix(altPath, actPath) { + return e.Miscpath + altPath[len(actPath):] + } + return altPath +} + +func (e *Environment) ToContainerPath(path string) string { + path = e.HostEnvironment.ToContainerPath(path) + actPath := filepath.Clean(e.Miscpath) + altPath := filepath.Clean(path) + if strings.HasPrefix(altPath, actPath) { + return "/private/tmp/act/" + altPath[len(actPath):] + } + return altPath +} + +func (e *Environment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return e.ExecWithCmdLine(command, "", env, user, workdir) +} + +func (e *Environment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +func (e *Environment) Start(b bool) common.Executor { + return e.HostEnvironment.Start(b).Then(func(ctx context.Context) error { + return e.start(ctx) + }) +} + +func (e *Environment) start(ctx context.Context) error { + actEnv := e.Env + + config := e.Config + + if config.AlwaysPull { + log.Printf("Pulling the latest version of %s...\n", actEnv.JobImage) + _, _, err := ExecWithEnv(ctx, nil, + "pull", actEnv.JobImage) + if err != nil { + return err + } + } + + log.Println("Cloning and configuring a new VM...") + vm, err := CreateNewVM(ctx, *actEnv, 0, 0) + if err != nil { + return err + } + var customDirectoryMounts []string + _ = os.MkdirAll(e.Miscpath, 0666) + customDirectoryMounts = append(customDirectoryMounts, "act:"+e.Miscpath) + e.vm = vm + err = vm.Start(config, actEnv, customDirectoryMounts) + if err != nil { + return err + } + + return e.execRaw(ctx, "ln -sf '/Volumes/My Shared Files/act' /private/tmp/act") +} +func (e *Environment) Stop(ctx context.Context) error { + log.Println("Stop VM?") + + actEnv := e.Env + + var vm *VM + if e.vm != nil { + vm = e.vm + } else { + vm = ExistingVM(*actEnv) + } + + if err := vm.Stop(); err != nil { + log.Printf("Failed to stop VM: %v", err) + } + + if err := vm.Delete(); err != nil { + log.Printf("Failed to delete VM: %v", err) + + return err + } + + return nil +} + +func (e *Environment) Remove() common.Executor { + return func(ctx context.Context) error { + _ = e.Stop(ctx) + log.Println("Remove VM?") + if e.CleanUp != nil { + e.CleanUp() + } + _ = os.RemoveAll(e.Path) + return e.Close()(ctx) + } +} +func (e *Environment) exec(ctx context.Context, command []string, _ string, env map[string]string, _, workdir string) error { + var wd string + if workdir != "" { + if filepath.IsAbs(workdir) { + wd = filepath.Clean(workdir) + } else { + wd = filepath.Clean(filepath.Join(e.Path, workdir)) + } + } else { + wd = e.ToContainerPath(e.Path) + } + envs := "" + for k, v := range env { + envs += shellquote.Join(k) + "=" + shellquote.Join(v) + " " + } + return e.execRaw(ctx, "cd "+shellquote.Join(wd)+"\nenv "+envs+shellquote.Join(command...)+"\nexit $?") +} + +func (e *Environment) execRaw(ctx context.Context, script string) error { + actEnv := e.Env + + var vm *VM + if e.vm != nil { + vm = e.vm + } else { + vm = ExistingVM(*actEnv) + } + + // Monitor "tart run" command's output so it's not silenced + go vm.MonitorTartRunOutput() + + config := e.Config + + ssh, err := vm.OpenSSH(ctx, config) + if err != nil { + return err + } + defer ssh.Close() + + session, err := ssh.NewSession() + if err != nil { + return err + } + defer session.Close() + + os.Stdout.WriteString(script + "\n") + + session.Stdin = strings.NewReader( + script, + ) + session.Stdout = e.StdOut + session.Stderr = e.StdOut + + err = session.Shell() + if err != nil { + return err + } + + return session.Wait() +} + +func (e *Environment) GetActPath() string { + return e.ToContainerPath(e.HostEnvironment.GetActPath()) +} + +func (e *Environment) Copy(destPath string, files ...*container.FileEntry) common.Executor { + return e.HostEnvironment.Copy(e.ToHostPath(destPath), files...) +} +func (e *Environment) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error { + return e.HostEnvironment.CopyTarStream(ctx, e.ToHostPath(destPath), tarStream) +} +func (e *Environment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return e.HostEnvironment.CopyDir(e.ToHostPath(destPath), srcPath, useGitIgnore) +} +func (e *Environment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + return e.HostEnvironment.GetContainerArchive(ctx, e.ToHostPath(srcPath)) +} + +func (e *Environment) GetRunnerContext(ctx context.Context) map[string]interface{} { + rctx := e.HostEnvironment.GetRunnerContext(ctx) + rctx["temp"] = e.ToContainerPath(e.TmpDir) + rctx["tool_cache"] = e.ToContainerPath(e.ToolCache) + return rctx +} diff --git a/pkg/tart/vm_darwin.go b/pkg/tart/vm_darwin.go new file mode 100644 index 00000000000..95fcc9bcc9e --- /dev/null +++ b/pkg/tart/vm_darwin.go @@ -0,0 +1,282 @@ +package tart + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/avast/retry-go" + "golang.org/x/crypto/ssh" +) + +const tartCommandName = "tart" + +var ( + ErrTartNotFound = errors.New("tart command not found") + ErrTartFailed = errors.New("tart command returned non-zero exit code") + ErrVMFailed = errors.New("VM errored") +) + +type VM struct { + id string + runcmd *exec.Cmd +} + +func ExistingVM(actEnv Env) *VM { + return &VM{ + id: actEnv.VirtualMachineID(), + } +} + +func CreateNewVM( + ctx context.Context, + actEnv Env, + cpuOverride uint64, + memoryOverride uint64, +) (*VM, error) { + log.Print("CreateNewVM") + vm := &VM{ + id: actEnv.VirtualMachineID(), + } + + if err := vm.cloneAndConfigure(ctx, actEnv, cpuOverride, memoryOverride); err != nil { + return nil, fmt.Errorf("failed to clone the VM: %w", err) + } + + return vm, nil +} + +func (vm *VM) cloneAndConfigure( + ctx context.Context, + actEnv Env, + cpuOverride uint64, + memoryOverride uint64, +) error { + _, _, err := Exec(ctx, "clone", actEnv.JobImage, vm.id) + if err != nil { + return err + } + + if cpuOverride != 0 { + _, _, err = Exec(ctx, "set", "--cpu", strconv.FormatUint(cpuOverride, 10), vm.id) + if err != nil { + return err + } + } + + if memoryOverride != 0 { + _, _, err = Exec(ctx, "set", "--memory", strconv.FormatUint(memoryOverride, 10), vm.id) + if err != nil { + return err + } + } + + return nil +} + +func (vm *VM) Start(config Config, _ *Env, customDirectoryMounts []string) error { + os.Remove(vm.tartRunOutputPath()) + var runArgs = []string{"run"} + + if config.Softnet { + runArgs = append(runArgs, "--net-softnet") + } + + if config.Headless { + runArgs = append(runArgs, "--no-graphics") + } + + for _, customDirectoryMount := range customDirectoryMounts { + runArgs = append(runArgs, "--dir", customDirectoryMount) + } + + runArgs = append(runArgs, vm.id) + + cmd := exec.Command(tartCommandName, runArgs...) + + outputFile, err := os.OpenFile(vm.tartRunOutputPath(), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + return err + } + _, _ = outputFile.WriteString(strings.Join(runArgs, " ") + "\n") + + cmd.Stdout = outputFile + cmd.Stderr = outputFile + + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + + err = cmd.Start() + if err != nil { + return err + } + vm.runcmd = cmd + return nil +} + +func (vm *VM) MonitorTartRunOutput() { + outputFile, err := os.Open(vm.tartRunOutputPath()) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to open VM's output file, "+ + "looks like the VM wasn't started in \"prepare\" step?\n") + + return + } + defer func() { + _ = outputFile.Close() + }() + + for { + n, err := io.Copy(os.Stdout, outputFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to display VM's output: %v\n", err) + + break + } + if n == 0 { + time.Sleep(100 * time.Millisecond) + + continue + } + } +} + +func (vm *VM) OpenSSH(ctx context.Context, config Config) (*ssh.Client, error) { + ip, err := vm.IP(ctx) + if err != nil { + return nil, err + } + addr := ip + ":22" + + var netConn net.Conn + if err := retry.Do(func() error { + dialer := net.Dialer{} + + netConn, err = dialer.DialContext(ctx, "tcp", addr) + + return err + }, retry.Context(ctx)); err != nil { + return nil, fmt.Errorf("%w: failed to connect via SSH: %v", ErrVMFailed, err) + } + + sshConfig := &ssh.ClientConfig{ + HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error { + return nil + }, + User: config.SSHUsername, + Auth: []ssh.AuthMethod{ + ssh.Password(config.SSHPassword), + }, + } + + sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, sshConfig) + if err != nil { + return nil, fmt.Errorf("%w: failed to connect via SSH: %v", ErrVMFailed, err) + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} + +func (vm *VM) IP(ctx context.Context) (string, error) { + stdout, _, err := Exec(ctx, "ip", "--wait", "60", vm.id) + if err != nil { + return "", err + } + + return strings.TrimSpace(stdout), nil +} + +func (vm *VM) Stop() error { + log.Println("Stop VM REAL?") + if vm.runcmd != nil { + log.Println("send sigint?") + _ = vm.runcmd.Process.Signal(os.Interrupt) + log.Println("wait?") + _ = vm.runcmd.Wait() + log.Println("wait done?") + return nil + } + _, _, err := Exec(context.Background(), "stop", vm.id) + return err +} + +func (vm *VM) Delete() error { + _, _, err := Exec(context.Background(), "delete", vm.id) + if err != nil { + return fmt.Errorf("%w: failed to delete VM %s: %v", ErrVMFailed, vm.id, err) + } + + return nil +} + +func Exec( + ctx context.Context, + args ...string, +) (string, string, error) { + return ExecWithEnv(ctx, nil, args...) +} + +func ExecWithEnv( + ctx context.Context, + env map[string]string, + args ...string, +) (string, string, error) { + cmd := exec.CommandContext(ctx, tartCommandName, args...) + + // Base environment + cmd.Env = cmd.Environ() + + // Environment overrides + for key, value := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value)) + } + + var stdout, stderr bytes.Buffer + + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return "", "", fmt.Errorf("%w: %s command not found in PATH, make sure Tart is installed", + ErrTartNotFound, tartCommandName) + } + + if _, ok := err.(*exec.ExitError); ok { + // Tart command failed, redefine the error + // to be the Tart-specific output + err = fmt.Errorf("%w: %q", ErrTartFailed, firstNonEmptyLine(stderr.String(), stdout.String())) + } + } + + return stdout.String(), stderr.String(), err +} + +func firstNonEmptyLine(outputs ...string) string { + for _, output := range outputs { + for _, line := range strings.Split(output, "\n") { + if line != "" { + return line + } + } + } + + return "" +} + +func (vm *VM) tartRunOutputPath() string { + return filepath.Join(os.TempDir(), fmt.Sprintf("%s-tart-run-output.log", vm.id)) +}