Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve S3 remote cache #187

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Improve S3 remote cache #187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a311ca6
Improve S3 remote cache
aledbf Jul 22, 2024
9434d06
Sync go work
aledbf Jul 22, 2024
68293ae
Cleanup
aledbf Jul 22, 2024
09d60b5
Ensure github.com/GeertJohan/go.rice exists
aledbf Jul 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions cmd/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,30 +306,30 @@ type pushOnlyRemoteCache struct {
C leeway.RemoteCache
}

func (c *pushOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
return c.C.ExistingPackages(pkgs)
func (c *pushOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
return c.C.ExistingPackages(ctx, pkgs)
}

func (c *pushOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
func (c *pushOnlyRemoteCache) Download(ctx context.Context, dst leeway.Cache, pkgs []*leeway.Package) error {
return nil
}

func (c *pushOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
return c.C.Upload(src, pkgs)
func (c *pushOnlyRemoteCache) Upload(ctx context.Context, src leeway.Cache, pkgs []*leeway.Package) error {
return c.C.Upload(ctx, src, pkgs)
}

type pullOnlyRemoteCache struct {
C leeway.RemoteCache
}

func (c *pullOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
return c.C.ExistingPackages(pkgs)
func (c *pullOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
return c.C.ExistingPackages(ctx, pkgs)
}

func (c *pullOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
return c.C.Download(dst, pkgs)
func (c *pullOnlyRemoteCache) Download(ctx context.Context, dst leeway.Cache, pkgs []*leeway.Package) error {
return c.C.Download(ctx, dst, pkgs)
}

func (c *pullOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
func (c *pullOnlyRemoteCache) Upload(ctx context.Context, src leeway.Cache, pkgs []*leeway.Package) error {
return nil
}
18 changes: 16 additions & 2 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,17 @@ import (
"runtime/trace"
"strings"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/gookit/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/xerrors"

"github.com/gitpod-io/leeway/pkg/leeway"

_ "github.com/GeertJohan/go.rice"
)

const (
Expand Down Expand Up @@ -183,7 +188,16 @@ func getRemoteCache() leeway.RemoteCache {
BucketName: remoteCacheBucket,
}
case "AWS":
rc, err := leeway.NewS3RemoteCache(remoteCacheBucket, nil)
ctx := context.Background()
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
log.Fatalf("cannot load AWS config: %v", err)
}

s3Client := s3.NewFromConfig(cfg)
stsClient := sts.NewFromConfig(cfg)

rc, err := leeway.NewS3RemoteCache(ctx, remoteCacheBucket, s3Client, stsClient)
if err != nil {
log.Fatalf("cannot access remote S3 cache: %v", err)
}
Expand All @@ -194,12 +208,12 @@ func getRemoteCache() leeway.RemoteCache {
BucketName: remoteCacheBucket,
}
}

}

return leeway.NoRemoteCache{}
}

// nolint: unused
func addExperimentalCommand(parent, child *cobra.Command) {
if os.Getenv("LEEWAY_EXPERIMENTAL") != "true" {
return
Expand Down
87 changes: 45 additions & 42 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,73 +1,76 @@
module github.com/gitpod-io/leeway

go 1.21
go 1.22

require (
github.com/aws/aws-sdk-go-v2 v1.21.2
github.com/aws/aws-sdk-go-v2/config v1.18.45
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
github.com/aws/aws-sdk-go-v2/service/sts v1.23.2
github.com/creack/pty v1.1.18
github.com/GeertJohan/go.rice v1.0.3
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/config v1.27.27
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3
github.com/creack/pty v1.1.21
github.com/disiqueira/gotree v1.0.0
github.com/dop251/goja v0.0.0-20231014103939-873a1496dc8e
github.com/fsnotify/fsnotify v1.6.0
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.3.1
github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5
github.com/fsnotify/fsnotify v1.7.0
github.com/google/go-cmp v0.5.5
github.com/google/uuid v1.6.0
github.com/gookit/color v1.5.4
github.com/imdario/mergo v0.3.13
github.com/in-toto/in-toto-golang v0.3.3
github.com/karrick/godirwalk v1.17.0
github.com/minio/highwayhash v1.0.2
github.com/opencontainers/runc v1.1.9
github.com/opencontainers/runtime-spec v1.1.0
github.com/segmentio/analytics-go/v3 v3.2.1
github.com/minio/highwayhash v1.0.3
github.com/opencontainers/runc v1.1.13
github.com/opencontainers/runtime-spec v1.2.0
github.com/segmentio/analytics-go/v3 v3.3.0
github.com/segmentio/textio v1.2.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.4
golang.org/x/mod v0.19.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
go.uber.org/mock v0.4.0
golang.org/x/mod v0.11.0
golang.org/x/sync v0.7.0
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9
gopkg.in/yaml.v3 v3.0.1
sigs.k8s.io/bom v0.1.0
)

require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
github.com/aws/smithy-go v1.15.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/daaku/go.zipexe v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/godbus/dbus/v5 v5.0.6 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/moby/sys/mountinfo v0.5.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
github.com/segmentio/backo-go v1.0.0 // indirect
github.com/shibumi/go-pathspec v1.3.0 // indirect
github.com/shibumi/go-pathspec v1.2.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.8.0 // indirect
sigs.k8s.io/release-utils v0.7.4-0.20230327115955-2b998c68e4b6 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.14.0 // indirect
sigs.k8s.io/release-utils v0.3.0 // indirect
)
Loading
Loading