diff --git a/go.mod b/go.mod index de3b0c6..e9df1b9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require github.com/google/go-cmp v0.6.0 require ( github.com/andybalholm/brotli v1.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/golang/snappy v0.0.2 // indirect @@ -16,7 +17,11 @@ require ( github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/ulikunitz/xz v0.5.9 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 03a23d1..067b7b3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -26,6 +28,12 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -35,3 +43,5 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 159787b..6f35187 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ import ( "github.com/drone/go-task/task" "github.com/drone/go-task/task/cloner" - "github.com/drone/go-task/task/download" + download "github.com/drone/go-task/task/downloader" "github.com/drone/go-task/task/drivers/cgi" ) diff --git a/task/download/download.go b/task/download/download.go deleted file mode 100644 index 0a5d8bc..0000000 --- a/task/download/download.go +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright 2024 Harness Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package download - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/drone/go-task/task" - "github.com/drone/go-task/task/cloner" - "github.com/drone/go-task/task/logger" - "github.com/mholt/archiver" -) - -// Downloader downloads a repository or a binary executable file -// It also takes care of where to download the repository or file -type Downloader interface { - // returns back the download directory - Download(context.Context, string, *task.Repository, *task.ExecutableConfig) (string, error) -} - -// New returns a downloader which downloads everything at the top-level -// dir directory -func New(cloner cloner.Cloner, dir string) Downloader { - return &downloader{cloner: cloner, dir: dir} -} - -type downloader struct { - cloner cloner.Cloner - dir string // top-level cache directory -} - -// getHashOfRepo constructs a hash from the repo config to figure out -// whether it should be re-cloned. -func getHashOfRepo(repo *task.Repository) string { - data := fmt.Sprintf("%s|%s|%s|%s", repo.Clone, repo.Ref, repo.Sha, repo.Download) - return getHash(data) -} - -func getHash(s string) string { - hash := sha256.New() - hash.Write([]byte(s)) - return hex.EncodeToString(hash.Sum(nil)) -} - -func (d *downloader) Download(ctx context.Context, taskType string, repo *task.Repository, exec *task.ExecutableConfig) (string, error) { - if exec != nil { - return d.handleDownloadExecutable(ctx, taskType, exec) - } else if repo != nil { - return d.handleDownloadRepo(ctx, repo) - } - return "", errors.New("no repository or executable urls provided to download") -} - -func (d *downloader) handleDownloadExecutable(ctx context.Context, taskType string, exec *task.ExecutableConfig) (string, error) { - operatingSystem := runtime.GOOS - architecture := runtime.GOARCH - url, ok := d.getExecutableUrl(exec, operatingSystem, architecture) - if !ok { - return "", fmt.Errorf("os [%s] and architecture [%s] are not specified in executable configuration", operatingSystem, architecture) - } - - dest := filepath.Join(d.getBaseDownloadDir(), taskType, exec.Version) - - if cacheHit := d.isCacheHit(ctx, dest); cacheHit { - // exit if the artifact destination already exists - return d.getDownloadPath(url, dest), nil - } - - // if no cache hit, remove all downloaded executables for this task's type - // so that we don't keep multiple executables of the same type - err := os.RemoveAll(filepath.Join(d.getBaseDownloadDir(), taskType)) - if err != nil { - return "", err - } - - binpath, err := d.downloadFile(ctx, url, dest) - if err != nil { - // remove the destination directory if downloading fails so it can be retried - os.RemoveAll(dest) - return "", err - } - d.logExecutableDownload(ctx, exec, operatingSystem, architecture) - - err = os.Chmod(binpath, 0777) - if err != nil { - return "", fmt.Errorf("failed to set executable flag in task file [%s]: %w", binpath, err) - } - return binpath, nil -} - -func (d *downloader) handleDownloadRepo(ctx context.Context, repo *task.Repository) (string, error) { - dest := d.getDownloadDir(repo) - - if cacheHit := d.isCacheHit(ctx, dest); cacheHit { - // exit if the artifact destination already exists - return dest, nil - } - - if repo.Download != "" { - return dest, d.downloadRepo(ctx, repo, dest) - } - return dest, d.clone(ctx, repo, dest) -} - -func (d *downloader) clone(ctx context.Context, repo *task.Repository, dest string) error { - log := logger.FromContext(ctx) - - // extract the clone url, ref and sha - url := repo.Clone - ref := repo.Ref - sha := repo.Sha - - log.With("source", url). - With("revision", ref). - With("sha", sha). - With("target", dest). - Debug("clone artifact") - - // clone the repository - err := d.cloner.Clone(ctx, cloner.Params{ - Repo: url, - Ref: ref, - Sha: sha, - Dir: dest, - }) - if err != nil { - return err - } - - return nil -} - -func (d *downloader) downloadRepo(ctx context.Context, repo *task.Repository, dest string) error { - log := logger.FromContext(ctx) - - downloadPath, err := d.downloadFile(ctx, repo.Download, dest) - if err != nil { - // remove the destination directory if downloading fails so it can be retried - os.RemoveAll(dest) - return err - } - - if err := d.unarchive(downloadPath, dest); err != nil { - // remove the destination directory if unarchiving fails so it can be retried - os.RemoveAll(dest) - return err - } - - log.With("source", repo.Download). - With("destination", dest). - Debug("extracted artifact") - - // delete the archive file after unpacking - os.Remove(downloadPath) - - return nil -} - -// unarchive unpacks srcPath into destDir. It unpacks everything directly into the -// destination directory and skips the top-level directory. -// For example, a github repo called "myrepo" with a file "task.yml" at the root -// will have an archive called "myrepo.zip" with the structure myrepo/task.yml. -// If destDir is "/tmp", this will extract the archive as /tmp/task.yml similar to the -// clone behavior. -func (d *downloader) unarchive(srcPath, destDir string) error { - // create a custom walk function - walkFn := func(f archiver.File) error { - // skip directories - if f.IsDir() { - return nil - } - - // get the relative path of the file within the archive - relPath := f.Name() - - // split the path into components - pathComponents := strings.Split(relPath, string(filepath.Separator)) - - // if there's more than one component, remove the first one (top-level directory) - if len(pathComponents) > 1 { - relPath = filepath.Join(pathComponents[1:]...) - } - - // construct the target file path - targetFile := filepath.Join(destDir, relPath) - - // ensure the directory structure exists - err := os.MkdirAll(filepath.Dir(targetFile), 0755) - if err != nil { - return fmt.Errorf("error creating directories: %w", err) - } - - // create the target file - outFile, err := os.Create(targetFile) - if err != nil { - return fmt.Errorf("error creating file: %w", err) - } - defer outFile.Close() - - // copy the contents from the archive to the new file - _, err = io.Copy(outFile, f) - if err != nil { - return fmt.Errorf("error copying file contents: %w", err) - } - - return nil - } - - // open and walk through the archive - err := archiver.Walk(srcPath, walkFn) - if err != nil { - return fmt.Errorf("error walking through archive: %w", err) - } - - return nil -} - -// downloadFile fetches the file from url and writes it to dest -func (d *downloader) downloadFile(ctx context.Context, url, dest string) (string, error) { - log := logger.FromContext(ctx) - - log.With("source", url). - With("destination", dest). - Debug("downloading artifact") - - // create the directory where the target is downloaded. - if err := os.MkdirAll(dest, 0777); err != nil { - return "", err - } - - // determine the file name and download location - downloadPath := d.getDownloadPath(url, dest) - - resp, err := http.Get(url) - if err != nil { - return "", fmt.Errorf("failed to download file: %w", err) - } - defer resp.Body.Close() - - if code := resp.StatusCode; code > 299 { - return "", fmt.Errorf("download error with status code %d", code) - } - - outFile, err := os.Create(downloadPath) - if err != nil { - return "", fmt.Errorf("failed to create file: %w", err) - } - defer outFile.Close() - - _, err = io.Copy(outFile, resp.Body) - if err != nil { - return "", fmt.Errorf("failed to write to file: %w", err) - } - - log.With("source", url). - With("destination", downloadPath). - Debug("downloaded artifact") - - return downloadPath, nil -} - -// getDownloadPath returns the full download path given the download url and the destination folder `dest` -func (d *downloader) getDownloadPath(url, dest string) string { - fileName := filepath.Base(url) - return filepath.Join(dest, fileName) -} - -// getDownloadDir returns the directory where the repository should be downloaded -// It joins the top-level directory with the hash of the repository config -func (d *downloader) getDownloadDir(repo *task.Repository) string { - return filepath.Join(d.getBaseDownloadDir(), getHashOfRepo(repo)) -} - -// getBaseDownloadDir returns the top-level directory where all files should be downloaded -func (d *downloader) getBaseDownloadDir() string { - return filepath.Join(d.dir, ".harness", "cache") -} - -// isCacheHit checks if the `dest` folder already exists -func (d *downloader) isCacheHit(ctx context.Context, dest string) bool { - log := logger.FromContext(ctx) - - if _, err := os.Stat(dest); err == nil { - log.With("target", dest). - Debug("cache hit") - return true - } - - log.With("target", dest). - Debug("cache miss") - return false -} - -// getExecutableUrl fetches the download url for a task's executable file, -// given the current system's operating system and architecture -func (d *downloader) getExecutableUrl(config *task.ExecutableConfig, operatingSystem, architecture string) (string, bool) { - for _, exec := range config.Executables { - if exec.Os == operatingSystem && exec.Arch == architecture { - return exec.Url, true - } - } - return "", false -} - -// logExecutableDownload writes details about the Executable struct used to download a task's executable file -func (d *downloader) logExecutableDownload(ctx context.Context, exec *task.ExecutableConfig, operatingSystem, architecture string) { - log := logger.FromContext(ctx) - filename := "executable_downloads.log" - file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Error(fmt.Sprintf("Failed to open log file [%s]: %v", filename, err)) - } - defer file.Close() - - // Convert the struct to JSON - data, err := json.Marshal(exec) - if err != nil { - log.Error(fmt.Sprintf("Failed to marshall Executable struct to json: %v", err)) - } - - entry := fmt.Sprintf("%s: dowloaded for os: [%s], arch: [%s] %s\n", time.Now().Format(time.RFC3339), operatingSystem, architecture, string(data)) - // Write the JSON string to the file, followed by a newline - if _, err := file.WriteString(entry); err != nil { - log.Error(fmt.Sprintf("Failed to write Executable struct to log file [%s]: %v", filename, err)) - } -} diff --git a/task/download/download_test.go b/task/download/download_test.go deleted file mode 100644 index f07e5cb..0000000 --- a/task/download/download_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2024 Harness Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package download diff --git a/task/download/util.go b/task/download/util.go deleted file mode 100644 index d914565..0000000 --- a/task/download/util.go +++ /dev/null @@ -1,46 +0,0 @@ -package download - -import ( - "net/url" - "os" - "strings" -) - -// getcache as a function for mocking -var getcache = os.UserCacheDir - -// ExpandCache returns the root directory where task -// downloads and repositories should be cached. -func ExpandCache(s string) string { - cache, _ := getcache() - return strings.ReplaceAll(s, "$XDG_CACHE_HOME", cache) -} - -// ExpandCacheSlice returns the root directory where task -// downloads and repositories should be cached. -func ExpandCacheSlice(items []string) []string { - for i, s := range items { - items[i] = ExpandCache(s) - } - return items -} - -// IsRepository returns true if the provided download url -// is a git repository. -func IsRepository(s string) bool { - u, _ := url.Parse(s) - return strings.HasSuffix(u.Path, ".git") -} - -// SplitRef splits the repository url and the commit ref. -func SplitRef(s string) (string, string) { - u, err := url.Parse(s) - if err != nil || u.Fragment == "" { - return s, "" - } else { - ref := u.Fragment - u.Fragment = "" - u.RawFragment = "" - return u.String(), ref - } -} diff --git a/task/download/util_test.go b/task/download/util_test.go deleted file mode 100644 index 3c531a5..0000000 --- a/task/download/util_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package download - -import ( - "os" - "testing" -) - -func TestExpandCache(t *testing.T) { - // provide a mock function to get the os cache - getcache = func() (string, error) { - return "/home/ubuntu/.cache", nil - } - // reset to the original when the test completes - defer func() { - getcache = os.UserCacheDir - }() - tests := []struct { - before string - after string - }{ - { - before: "$XDG_CACHE_HOME/harness/task/slack-v1.0.0", - after: "/home/ubuntu/.cache/harness/task/slack-v1.0.0", - }, - { - before: "/var/harness/cache/harness/task/slack-v1.0.0", - after: "/var/harness/cache/harness/task/slack-v1.0.0", - }, - } - for _, test := range tests { - if got, want := ExpandCache(test.before), test.after; got != want { - t.Errorf("Want cache dir %s, got %s", want, got) - } - } -} - -func TestSplitRef(t *testing.T) { - tests := []struct { - in string - url string - ref string - }{ - { - in: "https://github.com/octocat/hello-world.git#main", - url: "https://github.com/octocat/hello-world.git", - ref: "main", - }, - { - in: "https://github.com/octocat/hello-world.git", - url: "https://github.com/octocat/hello-world.git", - ref: "", - }, - } - for _, test := range tests { - url, ref := SplitRef(test.in) - if got, want := url, test.url; got != want { - t.Errorf("Expect url %s, got %s", got, want) - } - if got, want := ref, test.ref; got != want { - t.Errorf("Expect ref %s, got %s", got, want) - } - } -} - -func TestIsRepository(t *testing.T) { - tests := []struct { - url string - want bool - }{ - { - url: "https://github.com/octocat/hello-world.git", - want: true, - }, - { - url: "https://github.com/octocat/hello-world/downloads/latest/release.tar.gz", - want: false, - }, - } - for _, test := range tests { - if got, want := IsRepository(test.url), test.want; got != want { - t.Errorf("Expect %q is repository %v, got %v", test.url, got, want) - } - } -} diff --git a/task/downloader/downloader.go b/task/downloader/downloader.go new file mode 100644 index 0000000..5bfef52 --- /dev/null +++ b/task/downloader/downloader.go @@ -0,0 +1,42 @@ +// Copyright 2024 Harness Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package downloader + +import ( + "context" + "path/filepath" + + "github.com/drone/go-task/task" + "github.com/drone/go-task/task/cloner" +) + +// Downloader is an interface for structs +// that handle downloading a task's implementation + +type Downloader struct { + dir string + repoDownloader *repoDownloader + executableDownloader *executableDownloader +} + +func New(cloner cloner.Cloner, dir string) Downloader { + baseDir := getBaseDownloadDir(dir) + repoDownloader := newRepoDownloader(cloner) + executableDownloader := newExecutableDownloader() + return Downloader{dir: baseDir, repoDownloader: repoDownloader, executableDownloader: executableDownloader} +} + +// getBaseDownloadDir returns the top-level directory where all files should be downloaded +func getBaseDownloadDir(dir string) string { + return filepath.Join(dir, ".harness", "cache") +} + +func (d *Downloader) DownloadRepo(ctx context.Context, repo *task.Repository) (string, error) { + return d.repoDownloader.download(ctx, d.dir, repo) +} + +func (d *Downloader) DownloadExecutable(ctx context.Context, taskType string, version string, exec *task.ExecutableConfig) (string, error) { + return d.executableDownloader.download(ctx, d.dir, taskType, version, exec) +} diff --git a/task/downloader/executable_downloader.go b/task/downloader/executable_downloader.go new file mode 100644 index 0000000..b20c6ef --- /dev/null +++ b/task/downloader/executable_downloader.go @@ -0,0 +1,107 @@ +// Copyright 2024 Harness Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package downloader + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/drone/go-task/task" + "github.com/drone/go-task/task/logger" +) + +// removeAllFn as a function for mocking +var removeAllFn = os.RemoveAll + +// chmodFn as a function for mocking +var chmodFn = os.Chmod + +// executableDownloader a binary executable file +// It also takes care of where to download the file +type executableDownloader struct{} + +func newExecutableDownloader() *executableDownloader { + return &executableDownloader{} +} + +func (e *executableDownloader) download(ctx context.Context, dir string, taskType string, version string, exec *task.ExecutableConfig) (string, error) { + if exec == nil { + return "", errors.New("no executable urls provided to download") + } + operatingSystem := runtime.GOOS + architecture := runtime.GOARCH + url, ok := e.getExecutableUrl(exec, operatingSystem, architecture) + if !ok { + return "", fmt.Errorf("os [%s] and architecture [%s] are not specified in executable configuration", operatingSystem, architecture) + } + + destDir := filepath.Join(dir, taskType, version) + dest := getDownloadPath(url, destDir) + + if cacheHit := isCacheHitFn(ctx, destDir); cacheHit { + // exit if the artifact destination already exists + return dest, nil + } + + // if no cache hit, remove all downloaded executables for this task's type + // so that we don't keep multiple executables of the same type + err := removeAllFn(filepath.Join(dir, taskType)) + if err != nil { + return "", err + } + + binpath, err := downloadFileFn(ctx, url, dest) + if err != nil { + // remove the destination directory if downloading fails so it can be retried + removeAllFn(destDir) + return "", err + } + e.logExecutableDownload(ctx, exec, operatingSystem, architecture) + + err = chmodFn(binpath, 0777) + if err != nil { + return "", fmt.Errorf("failed to set executable flag in task file [%s]: %w", binpath, err) + } + return binpath, nil +} + +// getExecutableUrl fetches the download url for a task's executable file, +// given the current system's operating system and architecture +func (e *executableDownloader) getExecutableUrl(config *task.ExecutableConfig, operatingSystem, architecture string) (string, bool) { + for _, exec := range config.Executables { + if exec.Os == operatingSystem && exec.Arch == architecture { + return exec.Url, true + } + } + return "", false +} + +// logExecutableDownload writes details about the Executable struct used to download a task's executable file +func (e *executableDownloader) logExecutableDownload(ctx context.Context, exec *task.ExecutableConfig, operatingSystem, architecture string) { + log := logger.FromContext(ctx) + filename := "executable_downloads.log" + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Error(fmt.Sprintf("Failed to open log file [%s]: %v", filename, err)) + } + defer file.Close() + + // Convert the struct to JSON + data, err := json.Marshal(exec) + if err != nil { + log.Error(fmt.Sprintf("Failed to marshall Executable struct to json: %v", err)) + } + + entry := fmt.Sprintf("%s: dowloaded for os: [%s], arch: [%s] %s\n", time.Now().Format(time.RFC3339), operatingSystem, architecture, string(data)) + // Write the JSON string to the file, followed by a newline + if _, err := file.WriteString(entry); err != nil { + log.Error(fmt.Sprintf("Failed to write Executable struct to log file [%s]: %v", filename, err)) + } +} diff --git a/task/downloader/executable_downloader_test.go b/task/downloader/executable_downloader_test.go new file mode 100644 index 0000000..16aeb9b --- /dev/null +++ b/task/downloader/executable_downloader_test.go @@ -0,0 +1,177 @@ +// Copyright 2024 Harness Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package downloader + +import ( + "context" + "fmt" + "os" + "runtime" + "testing" + + "github.com/drone/go-task/task" + "github.com/stretchr/testify/assert" +) + +func TestDownloadExecutable(t *testing.T) { + // Save the original functions to restore them later + originalIsCacheHitFn := isCacheHitFn + defer func() { isCacheHitFn = originalIsCacheHitFn }() // Restore after the test + + originalDownloadFileFn := downloadFileFn + defer func() { downloadFileFn = originalDownloadFileFn }() // Restore after the test + + originalChmodFn := chmodFn + defer func() { chmodFn = originalChmodFn }() // Restore after the test + + originalRemoveAll := removeAllFn + defer func() { removeAllFn = originalRemoveAll }() // Restore after the test + removeAllFn = func(p string) error { + return nil + } + + downloader := newExecutableDownloader() + + tests := []struct { + name string + dir string + taskType string + version string + exec *task.ExecutableConfig + cacheHit bool + downloadErr bool + chmodErr bool + wantErr bool + }{ + { + name: "successful_download", + dir: "/tmp", + taskType: "binary", + version: "v1.0.0", + exec: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: runtime.GOOS, Arch: runtime.GOARCH, Url: "valid_url"}, + }, + }, + cacheHit: false, + wantErr: false, + }, + { + name: "cache_hit", + dir: "/tmp", + taskType: "binary", + version: "v1.0.0", + exec: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: runtime.GOOS, Arch: runtime.GOARCH, Url: "valid_url"}, + }, + }, + cacheHit: true, + wantErr: false, + }, + { + name: "download_error", + dir: "/tmp", + taskType: "binary", + version: "v1.0.0", + exec: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: runtime.GOOS, Arch: runtime.GOARCH, Url: "invalid_url"}, + }, + }, + cacheHit: false, + downloadErr: true, + wantErr: true, + }, + { + name: "chmod_error", + dir: "/tmp", + taskType: "binary", + version: "v1.0.0", + exec: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: runtime.GOOS, Arch: runtime.GOARCH, Url: "valid_url"}, + }, + }, + cacheHit: false, + chmodErr: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isCacheHitFn = func(ctx context.Context, dest string) bool { + return tt.cacheHit + } + + chmodFn = func(path string, mode os.FileMode) error { + if tt.chmodErr { + return fmt.Errorf("chmod error") + } + return nil + } + + downloadFileFn = func(ctx context.Context, url, dest string) (string, error) { + if tt.downloadErr { + return "", fmt.Errorf("download error") + } + return dest, nil + } + + _, err := downloader.download(context.Background(), tt.dir, tt.taskType, tt.version, tt.exec) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestGetExecutableUrl(t *testing.T) { + downloader := newExecutableDownloader() + + tests := []struct { + name string + config *task.ExecutableConfig + operatingSystem string + architecture string + expectedUrl string + found bool + }{ + { + name: "valid_executable", + config: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: "linux", Arch: "amd64", Url: "https://example.com/executable"}, + }, + }, + operatingSystem: "linux", + architecture: "amd64", + expectedUrl: "https://example.com/executable", + found: true, + }, + { + name: "invalid_executable", + config: &task.ExecutableConfig{ + Executables: []task.Executable{ + {Os: "linux", Arch: "amd64", Url: "https://example.com/executable"}, + }, + }, + operatingSystem: "windows", + architecture: "amd64", + expectedUrl: "", + found: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + url, found := downloader.getExecutableUrl(tt.config, tt.operatingSystem, tt.architecture) + assert.Equal(t, tt.expectedUrl, url) + assert.Equal(t, tt.found, found) + }) + } +} diff --git a/task/downloader/repo_downloader.go b/task/downloader/repo_downloader.go new file mode 100644 index 0000000..38d1d56 --- /dev/null +++ b/task/downloader/repo_downloader.go @@ -0,0 +1,182 @@ +// Copyright 2024 Harness Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package downloader + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/drone/go-task/task" + "github.com/drone/go-task/task/cloner" + "github.com/drone/go-task/task/logger" + "github.com/mholt/archiver" +) + +type repoDownloader struct { + cloner cloner.Cloner +} + +// repoDownloader downloads a repository +// It also takes care of where to download the repository +func newRepoDownloader(cloner cloner.Cloner) *repoDownloader { + return &repoDownloader{cloner: cloner} +} + +func (r *repoDownloader) download(ctx context.Context, dir string, repo *task.Repository) (string, error) { + if repo == nil { + return "", errors.New("no repository provided to download") + } + dest := r.getDownloadDir(dir, repo) + + if cacheHit := isCacheHitFn(ctx, dest); cacheHit { + // exit if the destination already exists + return dest, nil + } + + if repo.Download != "" { + return dest, r.downloadRepo(ctx, repo, dest) + } + return dest, r.clone(ctx, repo, dest) +} + +func (r *repoDownloader) clone(ctx context.Context, repo *task.Repository, dest string) error { + log := logger.FromContext(ctx) + + // extract the clone url, ref and sha + url := repo.Clone + ref := repo.Ref + sha := repo.Sha + + log.With("source", url). + With("revision", ref). + With("sha", sha). + With("target", dest). + Debug("clone artifact") + + // clone the repository + err := r.cloner.Clone(ctx, cloner.Params{ + Repo: url, + Ref: ref, + Sha: sha, + Dir: dest, + }) + if err != nil { + return err + } + + return nil +} + +func (r *repoDownloader) downloadRepo(ctx context.Context, repo *task.Repository, destDir string) error { + log := logger.FromContext(ctx) + + dest := getDownloadPath(repo.Download, destDir) + downloadPath, err := downloadFile(ctx, repo.Download, dest) + if err != nil { + // remove the destination directory if downloading fails so it can be retried + os.RemoveAll(dest) + return err + } + + if err := r.unarchive(downloadPath, dest); err != nil { + // remove the destination directory if unarchiving fails so it can be retried + os.RemoveAll(dest) + return err + } + + log.With("source", repo.Download). + With("destination", dest). + Debug("extracted artifact") + + // delete the archive file after unpacking + os.Remove(downloadPath) + + return nil +} + +// unarchive unpacks srcPath into destDir. It unpacks everything directly into the +// destination directory and skips the top-level directory. +// For example, a github repo called "myrepo" with a file "task.yml" at the root +// will have an archive called "myrepo.zip" with the structure myrepo/task.yml. +// If destDir is "/tmp", this will extract the archive as /tmp/task.yml similar to the +// clone behavior. +func (r *repoDownloader) unarchive(srcPath, destDir string) error { + // create a custom walk function + walkFn := func(f archiver.File) error { + // skip directories + if f.IsDir() { + return nil + } + + // get the relative path of the file within the archive + relPath := f.Name() + + // split the path into components + pathComponents := strings.Split(relPath, string(filepath.Separator)) + + // if there's more than one component, remove the first one (top-level directory) + if len(pathComponents) > 1 { + relPath = filepath.Join(pathComponents[1:]...) + } + + // construct the target file path + targetFile := filepath.Join(destDir, relPath) + + // ensure the directory structure exists + err := os.MkdirAll(filepath.Dir(targetFile), 0755) + if err != nil { + return fmt.Errorf("error creating directories: %w", err) + } + + // create the target file + outFile, err := os.Create(targetFile) + if err != nil { + return fmt.Errorf("error creating file: %w", err) + } + defer outFile.Close() + + // copy the contents from the archive to the new file + _, err = io.Copy(outFile, f) + if err != nil { + return fmt.Errorf("error copying file contents: %w", err) + } + + return nil + } + + // open and walk through the archive + err := archiver.Walk(srcPath, walkFn) + if err != nil { + return fmt.Errorf("error walking through archive: %w", err) + } + + return nil +} + +// getDownloadDir returns the directory where the repository should be downloaded +// It joins the top-level directory with the hash of the repository config +func (r *repoDownloader) getDownloadDir(dir string, repo *task.Repository) string { + return filepath.Join(dir, r.getHashOfRepo(repo)) +} + +// getHashOfRepo constructs a hash from the repo config to figure out +// whether it should be re-cloned. +func (r *repoDownloader) getHashOfRepo(repo *task.Repository) string { + data := fmt.Sprintf("%s|%s|%s|%s", repo.Clone, repo.Ref, repo.Sha, repo.Download) + return r.getHash(data) +} + +func (r *repoDownloader) getHash(s string) string { + hash := sha256.New() + hash.Write([]byte(s)) + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/task/downloader/repo_downloader_test.go b/task/downloader/repo_downloader_test.go new file mode 100644 index 0000000..81908f9 --- /dev/null +++ b/task/downloader/repo_downloader_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Harness Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package downloader + +import ( + "context" + "path/filepath" + "testing" + + "github.com/drone/go-task/task" + "github.com/drone/go-task/task/cloner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// Mock Cloner to use in tests +type MockCloner struct { + mock.Mock +} + +func (m *MockCloner) Clone(ctx context.Context, params cloner.Params) error { + args := m.Called(ctx, params) + return args.Error(0) +} + +// Test for download method +func TestDownload(t *testing.T) { + // Save the original httpGet to restore it later + originalIsCacheHitFn := isCacheHitFn + defer func() { isCacheHitFn = originalIsCacheHitFn }() // Restore after the test + + mockCloner := new(MockCloner) + downloader := newRepoDownloader(mockCloner) + + tests := []struct { + name string + repo *task.Repository + wantErr bool + cacheHit bool + cloneErr bool + }{ + { + name: "successful_clone", + repo: &task.Repository{Clone: "https://github.com/user/repo.git", Ref: "main"}, + wantErr: false, + cacheHit: false, + }, + { + name: "no_repository", + repo: nil, + wantErr: true, + cacheHit: false, + }, + { + name: "cache_hit", + repo: &task.Repository{Clone: "https://github.com/user/repo.git", Ref: "main"}, + wantErr: false, + cacheHit: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Mock the cache hit function + isCacheHitFn = func(ctx context.Context, dest string) bool { + return tt.cacheHit + } + if !tt.cacheHit { + mockCloner.On("Clone", mock.Anything, mock.Anything).Return(nil) + } + _, err := downloader.download(context.Background(), "/tmp", tt.repo) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestClone(t *testing.T) { + mockCloner := new(MockCloner) + downloader := newRepoDownloader(mockCloner) + + repo := &task.Repository{ + Clone: "https://github.com/user/repo.git", + Ref: "main", + Sha: "abc123", + } + + mockCloner.On("Clone", mock.Anything, mock.Anything).Return(nil).Once() + + err := downloader.clone(context.Background(), repo, "/tmp/clone-dir") + assert.NoError(t, err) + + mockCloner.AssertExpectations(t) +} + +func TestGetDownloadDir(t *testing.T) { + downloader := newRepoDownloader(nil) + + repo := &task.Repository{ + Clone: "https://github.com/user/repo.git", + Ref: "main", + Sha: "abc123", + } + + hash := downloader.getHashOfRepo(repo) + dir := downloader.getDownloadDir("/tmp", repo) + expectedDir := filepath.Join("/tmp", hash) + + assert.Equal(t, expectedDir, dir) +} diff --git a/task/downloader/util.go b/task/downloader/util.go new file mode 100644 index 0000000..4e75c17 --- /dev/null +++ b/task/downloader/util.go @@ -0,0 +1,124 @@ +package downloader + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/drone/go-task/task/logger" +) + +// functions for mocking +var ( + mkdirAllFn = os.MkdirAll + httpGetFn = http.Get + createFn = os.Create + copyFn = io.Copy + getcacheFn = os.UserCacheDir + isCacheHitFn = isCacheHit + downloadFileFn = downloadFile +) + +// downloadFile fetches the file from url and writes it to dest +func downloadFile(ctx context.Context, url, dest string) (string, error) { + log := logger.FromContext(ctx) + + log.With("source", url). + With("destination", dest). + Debug("downloading artifact") + + downloadDir := filepath.Dir(dest) + // create the directory where the target is downloaded. + if err := mkdirAllFn(downloadDir, 0777); err != nil { + return "", err + } + + resp, err := httpGetFn(url) + if err != nil { + return "", fmt.Errorf("failed to download file: %w", err) + } + defer resp.Body.Close() + + if code := resp.StatusCode; code > 299 { + return "", fmt.Errorf("download error with status code %d", code) + } + + outFile, err := createFn(dest) + if err != nil { + return "", fmt.Errorf("failed to create file: %w", err) + } + defer outFile.Close() + + _, err = copyFn(outFile, resp.Body) + if err != nil { + return "", fmt.Errorf("failed to write to file: %w", err) + } + + log.With("source", url). + With("destination", dest). + Debug("downloaded artifact") + + return dest, nil +} + +// getDownloadPath returns the full download path given the download url and the destination folder `dest` +func getDownloadPath(url, dest string) string { + fileName := filepath.Base(url) + return filepath.Join(dest, fileName) +} + +// isCacheHit checks if the `dest` folder already exists +func isCacheHit(ctx context.Context, dest string) bool { + log := logger.FromContext(ctx) + + if _, err := os.Stat(dest); err == nil { + log.With("target", dest). + Debug("cache hit") + return true + } + + log.With("target", dest). + Debug("cache miss") + return false +} + +// ExpandCache returns the root directory where task +// downloads and repositories should be cached. +func ExpandCache(s string) string { + cache, _ := getcacheFn() + return strings.ReplaceAll(s, "$XDG_CACHE_HOME", cache) +} + +// ExpandCacheSlice returns the root directory where task +// downloads and repositories should be cached. +func ExpandCacheSlice(items []string) []string { + for i, s := range items { + items[i] = ExpandCache(s) + } + return items +} + +// IsRepository returns true if the provided download url +// is a git repository. +func IsRepository(s string) bool { + u, _ := url.Parse(s) + return strings.HasSuffix(u.Path, ".git") +} + +// SplitRef splits the repository url and the commit ref. +func SplitRef(s string) (string, string) { + u, err := url.Parse(s) + if err != nil || u.Fragment == "" { + return s, "" + } else { + ref := u.Fragment + u.Fragment = "" + u.RawFragment = "" + return u.String(), ref + } +} diff --git a/task/downloader/util_test.go b/task/downloader/util_test.go new file mode 100644 index 0000000..caa4418 --- /dev/null +++ b/task/downloader/util_test.go @@ -0,0 +1,237 @@ +package downloader + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "strings" + "testing" +) + +func TestDownloadFile(t *testing.T) { + // Save the original functions + originalmkdirAll := mkdirAllFn + originalHttpGet := httpGetFn + originalCreateFn := createFn + originalCopyFn := copyFn + + // Mock functions + mkdirAllFn = func(s string, m os.FileMode) error { + return nil + } + copyFn = func(w io.Writer, r io.Reader) (int64, error) { + return 0, nil + } + + // Restore original functions when test finishes + defer func() { mkdirAllFn = originalmkdirAll }() + defer func() { httpGetFn = originalHttpGet }() + defer func() { createFn = originalCreateFn }() + defer func() { copyFn = originalCopyFn }() + + tests := []struct { + name string + url string + dest string + fileCreateErr bool + wantErr bool + mockGetFn func(string) (*http.Response, error) + }{ + { + name: "successful_download", + url: "http://example.com/file.txt", + dest: "/tmp/testfile.txt", + wantErr: false, + mockGetFn: func(url string) (*http.Response, error) { + body := io.NopCloser(strings.NewReader("mock file content")) + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + }, nil + }, + }, + { + name: "http_error", + url: "http://example.com/nonexistent", + dest: "/tmp/testfile.txt", + wantErr: true, + mockGetFn: func(url string) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader("")), + }, nil + }, + }, + { + name: "file_creation_error", + url: "http://example.com/file.txt", + dest: "/invalid/dir/testfile.txt", + fileCreateErr: true, + wantErr: true, + mockGetFn: func(url string) (*http.Response, error) { + body := io.NopCloser(strings.NewReader("mock file content")) + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + }, nil + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Mock the functions + httpGetFn = tt.mockGetFn + if tt.fileCreateErr { + createFn = func(s string) (*os.File, error) { + return nil, fmt.Errorf("error creating file") + } + } else { + createFn = func(s string) (*os.File, error) { + return &os.File{}, nil + } + } + + _, err := downloadFile(context.Background(), tt.url, tt.dest) + if gotErr := err != nil; gotErr != tt.wantErr { + t.Errorf("downloadFile() error = %v, wantErr %v", gotErr, tt.wantErr) + } + }) + } +} + +func TestGetDownloadPath(t *testing.T) { + tests := []struct { + url string + dest string + expected string + }{ + { + url: "http://example.com/file.txt", + dest: "/downloads", + expected: "/downloads/file.txt", + }, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("url: %s", tt.url), func(t *testing.T) { + if got := getDownloadPath(tt.url, tt.dest); got != tt.expected { + t.Errorf("getDownloadPath(%q, %q) = %q, want %q", tt.url, tt.dest, got, tt.expected) + } + }) + } +} + +func TestIsCacheHit(t *testing.T) { + tests := []struct { + name string + dest string + setup func(string) // function to create a file for "cache hit" + wantHit bool + }{ + { + name: "cache_hit", + dest: "/tmp/testfile.txt", + setup: func(dest string) { + _, _ = os.Create(dest) + }, + wantHit: true, + }, + { + name: "cache_miss", + dest: "/tmp/nonexistent.txt", + setup: func(dest string) {}, // no setup for cache miss + wantHit: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup(tt.dest) + + if got := isCacheHit(context.Background(), tt.dest); got != tt.wantHit { + t.Errorf("isCacheHit() = %v, want %v", got, tt.wantHit) + } + }) + } +} + +func TestExpandCache(t *testing.T) { + // provide a mock function to get the os cache + getcacheFn = func() (string, error) { + return "/home/ubuntu/.cache", nil + } + // reset to the original when the test completes + defer func() { + getcacheFn = os.UserCacheDir + }() + tests := []struct { + before string + after string + }{ + { + before: "$XDG_CACHE_HOME/harness/task/slack-v1.0.0", + after: "/home/ubuntu/.cache/harness/task/slack-v1.0.0", + }, + { + before: "/var/harness/cache/harness/task/slack-v1.0.0", + after: "/var/harness/cache/harness/task/slack-v1.0.0", + }, + } + for _, test := range tests { + if got, want := ExpandCache(test.before), test.after; got != want { + t.Errorf("Want cache dir %s, got %s", want, got) + } + } +} + +func TestSplitRef(t *testing.T) { + tests := []struct { + in string + url string + ref string + }{ + { + in: "https://github.com/octocat/hello-world.git#main", + url: "https://github.com/octocat/hello-world.git", + ref: "main", + }, + { + in: "https://github.com/octocat/hello-world.git", + url: "https://github.com/octocat/hello-world.git", + ref: "", + }, + } + for _, test := range tests { + url, ref := SplitRef(test.in) + if got, want := url, test.url; got != want { + t.Errorf("Expect url %s, got %s", got, want) + } + if got, want := ref, test.ref; got != want { + t.Errorf("Expect ref %s, got %s", got, want) + } + } +} + +func TestIsRepository(t *testing.T) { + tests := []struct { + url string + want bool + }{ + { + url: "https://github.com/octocat/hello-world.git", + want: true, + }, + { + url: "https://github.com/octocat/hello-world/downloads/latest/release.tar.gz", + want: false, + }, + } + for _, test := range tests { + if got, want := IsRepository(test.url), test.want; got != want { + t.Errorf("Expect %q is repository %v, got %v", test.url, got, want) + } + } +} diff --git a/task/drivers/cgi/driver.go b/task/drivers/cgi/driver.go index 1d4cddb..b86b2b9 100644 --- a/task/drivers/cgi/driver.go +++ b/task/drivers/cgi/driver.go @@ -11,7 +11,7 @@ import ( "github.com/drone/go-task/task" "github.com/drone/go-task/task/builder" - "github.com/drone/go-task/task/download" + "github.com/drone/go-task/task/downloader" "github.com/drone/go-task/task/logger" ) @@ -30,12 +30,12 @@ type Config struct { } // New returns the task execution driver. -func New(d download.Downloader) task.Handler { +func New(d downloader.Downloader) task.Handler { return &driver{downloader: d} } type driver struct { - downloader download.Downloader + downloader downloader.Downloader } // Handle handles the task execution request. @@ -51,7 +51,7 @@ func (d *driver) Handle(ctx context.Context, req *task.Request) task.Response { return task.Error(err) } - path, err := d.downloader.Download(ctx, req.Task.Type, conf.Repository, conf.ExecutableConfig) + path, err := d.downloader.DownloadRepo(ctx, conf.Repository) if err != nil { log.With("error", err).Error("artifact download failed") return task.Error(err) diff --git a/task/types.go b/task/types.go index e8d73ca..45f89e1 100644 --- a/task/types.go +++ b/task/types.go @@ -68,7 +68,6 @@ type Repository struct { // supported operating systems and architectures type ExecutableConfig struct { Executables []Executable `json:"executables"` - Version string `json:"version"` } // Executable provides the url to download