diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml deleted file mode 100644 index 5b4e581..0000000 --- a/.github/workflows/codesee-arch-diagram.yml +++ /dev/null @@ -1,81 +0,0 @@ -on: - push: - branches: - - master - pull_request_target: - types: [opened, synchronize, reopened] - -name: CodeSee Map - -jobs: - test_map_action: - runs-on: ubuntu-latest - continue-on-error: true - name: Run CodeSee Map Analysis - steps: - - name: checkout - id: checkout - uses: actions/checkout@v2 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - fetch-depth: 0 - - # codesee-detect-languages has an output with id languages. - - name: Detect Languages - id: detect-languages - uses: Codesee-io/codesee-detect-languages-action@latest - - - name: Configure JDK 16 - uses: actions/setup-java@v2 - if: ${{ fromJSON(steps.detect-languages.outputs.languages).java }} - with: - java-version: '16' - distribution: 'zulu' - - # CodeSee Maps Go support uses a static binary so there's no setup step required. - - - name: Configure Node.js 14 - uses: actions/setup-node@v2 - if: ${{ fromJSON(steps.detect-languages.outputs.languages).javascript }} - with: - node-version: '14' - - - name: Configure Python 3.x - uses: actions/setup-python@v2 - if: ${{ fromJSON(steps.detect-languages.outputs.languages).python }} - with: - python-version: '3.x' - architecture: 'x64' - - - name: Configure Ruby '3.x' - uses: ruby/setup-ruby@v1 - if: ${{ fromJSON(steps.detect-languages.outputs.languages).ruby }} - with: - ruby-version: '3.0' - - # CodeSee Maps Rust support uses a static binary so there's no setup step required. - - - name: Generate Map - id: generate-map - uses: Codesee-io/codesee-map-action@latest - with: - step: map - github_ref: ${{ github.ref }} - languages: ${{ steps.detect-languages.outputs.languages }} - - - name: Upload Map - id: upload-map - uses: Codesee-io/codesee-map-action@latest - with: - step: mapUpload - api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} - github_ref: ${{ github.ref }} - - - name: Insights - id: insights - uses: Codesee-io/codesee-map-action@latest - with: - step: insights - api_token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} - github_ref: ${{ github.ref }} diff --git a/cli/cmd/module.go b/cli/cmd/module.go index 257e409..25502f8 100644 --- a/cli/cmd/module.go +++ b/cli/cmd/module.go @@ -6,6 +6,7 @@ import ( "os" "github.com/dotindustries/moar/moarpb" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" ) @@ -15,6 +16,42 @@ var moduleCmd = &cobra.Command{ Aliases: []string{"m"}, } +var getAll bool +var getModuleCmd = &cobra.Command{ + Use: "get", + Short: "Get module details", + Aliases: []string{"g", "read", "r"}, + Run: func(cmd *cobra.Command, args []string) { + client := protobufClient() + + request := &moarpb.GetModuleRequest{} + + if !getAll { + if len(args) < 1 { + fmt.Println("ERROR: module name not provided") + os.Exit(1) + } + request.ModuleName = args[0] + } + response, err := client.GetModule(context.Background(), request) + if err != nil { + fmt.Println("ERROR: ", err) + os.Exit(1) + } + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"Module", "Author", "Language", "Version"}) + rowConfigAutoMerge := table.RowConfig{AutoMerge: true} + for _, module := range response.Module { + for _, v := range module.Versions { + t.AppendRow(table.Row{module.Name, module.Author, module.Language, v.Name}, rowConfigAutoMerge) + // t.AppendSeparator() + } + } + t.Render() + }, +} + var author, language string var newModuleCmd = &cobra.Command{ @@ -59,5 +96,8 @@ func init() { panic(err) } moduleCmd.AddCommand(newModuleCmd) + + getModuleCmd.Flags().BoolVarP(&getAll, "all", "a", false, "Gets all modules within the system ignoring provided module name arguments") + moduleCmd.AddCommand(getModuleCmd) rootCmd.AddCommand(moduleCmd) } diff --git a/cli/cmd/up.go b/cli/cmd/up.go index b4bc1c5..b591e4c 100644 --- a/cli/cmd/up.go +++ b/cli/cmd/up.go @@ -4,7 +4,7 @@ import ( "net/http" "os" - s32 "github.com/dotindustries/moar/internal/storage/s3" + "github.com/dotindustries/moar/internal/storage/s3" "github.com/gorilla/handlers" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -32,7 +32,7 @@ var upCmd = &cobra.Command{ // case "etcd": // moduleStorage = internal.NewDatabase() case "s3": - moduleStorage = s32.New(storageAddress) + moduleStorage = s3.New(storageAddress) default: logrus.Fatalf("invalid module storage type: '%s'", moduleStorageType) } @@ -56,7 +56,7 @@ var upCmd = &cobra.Command{ func init() { upCmd.Flags().StringVar(&moduleStorageType, "storage_type", "s3", "Defines what storage type to use. Possible values: s3") - upCmd.Flags().StringVar(&storageAddress, "storage_addr", "localhost:9000", "The address to reach the storage") + upCmd.Flags().StringVar(&storageAddress, "storage_addr", "", "The address to reach the storage") upCmd.Flags().StringVar(&host, "host", ":8000", "The address to bind the server to") upCmd.Flags().BoolVar(&versionOverwriteEnabled, "overwrite", false, "Toggles whether version overwrite is enabled") rootCmd.AddCommand(upCmd) diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..673dbe0 --- /dev/null +++ b/docker/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=moar \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 88784b8..1baa787 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -66,6 +66,7 @@ services: image: dotindustries/moar-registry pull_policy: always hostname: moar + restart: on-failure command: - ./moar - up diff --git a/go.mod b/go.mod index 2948cf1..6e5c338 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/Masterminds/semver v1.5.0 github.com/gorilla/handlers v1.5.1 + github.com/jedib0t/go-pretty/v6 v6.2.4 github.com/minio/minio-go/v7 v7.0.14 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index eff395e..2ac8039 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,7 @@ github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8S github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -160,6 +161,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jcchavezs/porto v0.1.0 h1:Xmxxn25zQMmgE7/yHYmh19KcItG81hIwfbEEFnd6w/Q= github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= +github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk= +github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= @@ -192,6 +195,8 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= @@ -226,6 +231,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -398,6 +404,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/registry/service.go b/internal/registry/service.go index 0afb105..e759c52 100644 --- a/internal/registry/service.go +++ b/internal/registry/service.go @@ -12,6 +12,7 @@ import ( type Reader interface { ModuleResources(ctx context.Context, module string, version string, data bool) ([]internal.File, error) GetModule(ctx context.Context, name string, loadData bool) (*internal.Module, error) + GetModules(ctx context.Context, loadData bool) ([]*internal.Module, error) Close() error } @@ -97,6 +98,14 @@ func (s *Service) GetModule(ctx context.Context, name string, loadData bool) (*i return m, err } +func (s *Service) GetAllModules(ctx context.Context, loadData bool) ([]*internal.Module, error) { + modules, err := s.storage.GetModules(ctx, loadData) + if err != nil { + return nil, err + } + return modules, nil +} + func (s *Service) Close() error { return s.storage.Close() } diff --git a/internal/storage/s3/setup.go b/internal/storage/s3/setup.go new file mode 100644 index 0000000..fa2a2b4 --- /dev/null +++ b/internal/storage/s3/setup.go @@ -0,0 +1,65 @@ +package s3 + +import ( + "os" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +const defaultBucket = "modules" +const defaultEndpoint = "localhost:9000" + +func New(endpoint string) *Storage { + logger := logrus.WithField("op", "storage") + endpoint = validateEndpoint(endpoint) + creds := collectCredentials() + useSSL := false + if os.Getenv("S3_USE_SSL") != "" { + useSSL = true + } + minioClient, err := minio.New(endpoint, + &minio.Options{ + Creds: creds, + Secure: useSSL, + }, + ) + if err != nil { + logger.Fatalln(err) + } + + s := &Storage{ + minioClient: minioClient, + logger: logger, + bucket: bucket(), + } + s.setup() + return s +} + +func collectCredentials() *credentials.Credentials { + accessKeyID := os.Getenv("S3_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("S3_SECRET_ACCESS_KEY") + sessionToken := os.Getenv("AWS_SESSION_TOKEN") + return credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken) +} + +func bucket() string { + bucket := os.Getenv("S3_BUCKET") + if bucket == "" { + bucket = defaultBucket + } + return bucket +} + +func validateEndpoint(endpoint string) string { + if endpoint == "" { + endpoint = os.Getenv("S3_ENDPOINT_URL") + if endpoint == "" { + endpoint = defaultEndpoint + } + } + logrus.Infof("Using S3 at %s", endpoint) + return endpoint +} diff --git a/internal/storage/s3/storage.go b/internal/storage/s3/storage.go index 93bfca0..5aa31d0 100644 --- a/internal/storage/s3/storage.go +++ b/internal/storage/s3/storage.go @@ -6,47 +6,22 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path" + "strings" "github.com/dotindustries/moar/internal" "github.com/dotindustries/moar/internal/storage" "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "github.com/sirupsen/logrus" ) -const modulesBucket = "modules" const manifestSuffix = "/manifest.json" type Storage struct { - endpoint string minioClient *minio.Client logger *logrus.Entry -} - -func New(endpoint string) *Storage { - logger := logrus.WithField("op", "storage") - if endpoint == "" { - endpoint = "localhost:9000" - } - accessKeyID := "minio" - secretAccessKey := "minio123" - useSSL := false - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: useSSL, - }) - if err != nil { - logger.Fatalln(err) - } - - s := &Storage{ - endpoint: endpoint, - minioClient: minioClient, - logger: logger, - } - s.setup() - return s + bucket string } func (s *Storage) PutModule(ctx context.Context, module internal.Module) error { @@ -64,7 +39,7 @@ func (s *Storage) PutModule(ctx context.Context, module internal.Module) error { objectName := moduleManifestObjectName(module.Name) _, err = s.minioClient.PutObject( ctx, - modulesBucket, + s.bucket, objectName, reader, int64(reader.Len()), @@ -80,7 +55,7 @@ func (s *Storage) RemoveModule(ctx context.Context, name string) error { objectName := moduleManifestObjectNameFromString(name) err := s.minioClient.RemoveObject( ctx, - modulesBucket, + s.bucket, objectName, minio.RemoveObjectOptions{ ForceDelete: true, @@ -100,7 +75,7 @@ func (s *Storage) PutVersion(ctx context.Context, module string, version string, // rollback function in case of errors if err != nil { for _, objectName := range added { - rollBackErr := s.minioClient.RemoveObject(ctx, modulesBucket, objectName, minio.RemoveObjectOptions{ForceDelete: true}) + rollBackErr := s.minioClient.RemoveObject(ctx, s.bucket, objectName, minio.RemoveObjectOptions{ForceDelete: true}) if rollBackErr != nil { s.logger.Errorf("Failed to roll back after failing to put stylesheet: %s", rollBackErr) // update err to rollback error @@ -114,7 +89,7 @@ func (s *Storage) PutVersion(ctx context.Context, module string, version string, objectName := basePath + file.Name info, err := s.minioClient.PutObject( ctx, - modulesBucket, + s.bucket, objectName, r, r.Size(), @@ -135,7 +110,7 @@ func (s *Storage) RemoveVersion(ctx context.Context, module string, version stri basePath := moduleVersionBasePath(module, version) return s.minioClient.RemoveObject( ctx, - modulesBucket, + s.bucket, basePath, minio.RemoveObjectOptions{ ForceDelete: true, @@ -147,13 +122,13 @@ func (s *Storage) ModuleResources(ctx context.Context, module string, version st basePath := moduleVersionBasePath(module, version) var files []internal.File - for object := range s.minioClient.ListObjects(ctx, modulesBucket, minio.ListObjectsOptions{ + for object := range s.minioClient.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{ Prefix: basePath, Recursive: true, }) { var bts []byte if loadData { - obj, err := s.minioClient.GetObject(ctx, modulesBucket, object.Key, minio.GetObjectOptions{}) + obj, err := s.minioClient.GetObject(ctx, s.bucket, object.Key, minio.GetObjectOptions{}) if err != nil { return nil, err } @@ -166,7 +141,7 @@ func (s *Storage) ModuleResources(ctx context.Context, module string, version st return nil, err } } - stat, err := s.minioClient.StatObject(ctx, modulesBucket, object.Key, minio.StatObjectOptions{}) + stat, err := s.minioClient.StatObject(ctx, s.bucket, object.Key, minio.StatObjectOptions{}) if err != nil { return nil, err } @@ -174,14 +149,14 @@ func (s *Storage) ModuleResources(ctx context.Context, module string, version st Name: path.Base(object.Key), MimeType: stat.ContentType, Data: bts, - Uri: fmt.Sprintf("%s/%s", modulesBucket, object.Key), + Uri: fmt.Sprintf("%s/%s", s.bucket, object.Key), }) } return files, nil } func (s *Storage) checkObjectExists(ctx context.Context, objectName string) bool { - _, err := s.minioClient.StatObject(ctx, modulesBucket, objectName, minio.StatObjectOptions{}) + _, err := s.minioClient.StatObject(ctx, s.bucket, objectName, minio.StatObjectOptions{}) if err != nil { return false } @@ -190,7 +165,26 @@ func (s *Storage) checkObjectExists(ctx context.Context, objectName string) bool func (s *Storage) GetModule(ctx context.Context, name string, loadData bool) (*internal.Module, error) { objectName := moduleManifestObjectNameFromString(name) - manifestObj, err := s.minioClient.GetObject(ctx, modulesBucket, objectName, minio.GetObjectOptions{}) + return s.loadModule(ctx, objectName, loadData) +} + +func (s *Storage) GetModules(ctx context.Context, loadData bool) (modules []*internal.Module, err error) { + for object := range s.minioClient.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Recursive: true}) { + isManifestObj := strings.HasSuffix(object.Key, manifestSuffix) + if isManifestObj { + var module *internal.Module + module, err = s.loadModule(ctx, object.Key, loadData) + if err != nil { + return nil, err + } + modules = append(modules, module) + } + } + return +} + +func (s *Storage) loadModule(ctx context.Context, objectName string, loadData bool) (*internal.Module, error) { + manifestObj, err := s.minioClient.GetObject(ctx, s.bucket, objectName, minio.GetObjectOptions{}) if err != nil { return nil, err } @@ -222,25 +216,42 @@ func (s *Storage) Close() error { } func (s *Storage) setup() { - exists, err := s.minioClient.BucketExists(context.Background(), modulesBucket) + exists, err := s.minioClient.BucketExists(context.Background(), s.bucket) if err != nil { s.logger.Fatal(err) } if !exists { - err = s.minioClient.MakeBucket(context.Background(), modulesBucket, minio.MakeBucketOptions{}) + err = s.minioClient.MakeBucket(context.Background(), s.bucket, minio.MakeBucketOptions{}) if err != nil { s.logger.Fatal(err) } - readOnlyPolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + modulesBucket + `/*"],"Sid": ""}]}` - err = s.minioClient.SetBucketPolicy(context.Background(), modulesBucket, readOnlyPolicy) + s.logger.Info("Modules bucket setup complete") + } else { + s.logger.Info("Modules bucket verified") + } + + policy, err := s.minioClient.GetBucketPolicy(context.Background(), s.bucket) + if err != nil { + return + } + if policy == "" { + readOnlyPolicy := s.readOnlyPolicy() + s.logger.Info("Updating bucket policy from '", policy, "' to '", readOnlyPolicy, "'") + err = s.minioClient.SetBucketPolicy(context.Background(), s.bucket, readOnlyPolicy) if err != nil { s.logger.Fatal(err) return } - s.logger.Info("Modules bucket setup complete") - } else { - s.logger.Info("Modules bucket verified") + s.logger.Info("Bucket policy updated") + } +} + +func (s *Storage) readOnlyPolicy() string { + if customPolicy := os.Getenv("S3_BUCKET_POLICY"); customPolicy != "" { + return customPolicy } + // minio and AWS default + return `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + s.bucket + `/*"],"Sid": ""}]}` } func moduleManifestObjectName(module string) string { diff --git a/moar.proto b/moar.proto index 34f5b03..d994ce1 100644 --- a/moar.proto +++ b/moar.proto @@ -18,7 +18,7 @@ message GetModuleRequest { } message GetModuleResponse { - Module module = 1; + repeated Module module = 1; } message GetUrlRequest { diff --git a/moarpb/moar.pb.go b/moarpb/moar.pb.go index 05b10d2..1e1d286 100644 --- a/moarpb/moar.pb.go +++ b/moarpb/moar.pb.go @@ -72,7 +72,7 @@ type GetModuleResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Module *Module `protobuf:"bytes,1,opt,name=module,proto3" json:"module,omitempty"` + Module []*Module `protobuf:"bytes,1,rep,name=module,proto3" json:"module,omitempty"` } func (x *GetModuleResponse) Reset() { @@ -107,7 +107,7 @@ func (*GetModuleResponse) Descriptor() ([]byte, []int) { return file_moar_proto_rawDescGZIP(), []int{1} } -func (x *GetModuleResponse) GetModule() *Module { +func (x *GetModuleResponse) GetModule() []*Module { if x != nil { return x.Module } @@ -906,7 +906,7 @@ var file_moar_proto_rawDesc = []byte{ 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x3b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, - 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, + 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6d, 0x6f, 0x61, 0x72, 0x70, 0x62, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x22, 0x8e, 0x01, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x75, 0x6c, @@ -1006,10 +1006,10 @@ var file_moar_proto_rawDesc = []byte{ 0x72, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x6f, 0x61, 0x72, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x20, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x61, 0x64, 0x69, 0x6c, 0x61, 0x73, 0x2f, 0x6d, 0x6f, - 0x61, 0x72, 0x2f, 0x6d, 0x6f, 0x61, 0x72, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6f, 0x74, 0x69, 0x6e, 0x64, 0x75, 0x73, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x61, 0x72, 0x2f, 0x6d, 0x6f, 0x61, 0x72, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/moarpb/moar.twirp.go b/moarpb/moar.twirp.go index 9e1bce0..13c258b 100644 --- a/moarpb/moar.twirp.go +++ b/moarpb/moar.twirp.go @@ -2486,44 +2486,44 @@ func callClientError(ctx context.Context, h *twirp.ClientHooks, err twirp.Error) } var twirpFileDescriptor0 = []byte{ - // 609 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0x4d, 0x6f, 0xd3, 0x4c, - 0x10, 0x7e, 0x1d, 0xe7, 0x75, 0x93, 0x49, 0x9a, 0x34, 0xdb, 0x36, 0x35, 0x06, 0xaa, 0xc8, 0x07, - 0x14, 0x09, 0x29, 0x91, 0x82, 0x2a, 0x0e, 0x5c, 0xa0, 0x45, 0x14, 0x10, 0x20, 0x64, 0x68, 0x25, - 0xb8, 0x6d, 0x92, 0x21, 0xb5, 0xe4, 0x78, 0xc3, 0xee, 0x1a, 0xa9, 0x07, 0xfe, 0x02, 0x77, 0x7e, - 0x0c, 0xff, 0x0d, 0xd9, 0xbb, 0xfe, 0xc4, 0x41, 0xad, 0xc4, 0x6d, 0x77, 0xbe, 0x9e, 0x67, 0x9f, - 0x99, 0xb1, 0x01, 0xd6, 0x8c, 0xf2, 0xc9, 0x86, 0x33, 0xc9, 0x88, 0x15, 0x9f, 0x37, 0x73, 0x77, - 0x06, 0x7b, 0xe7, 0x28, 0xdf, 0xb2, 0x65, 0x14, 0xa0, 0x87, 0x5f, 0x23, 0x14, 0x92, 0x1c, 0xc7, - 0x91, 0xb1, 0xe1, 0x1d, 0x5d, 0xa3, 0x6d, 0x8c, 0x8c, 0x71, 0xdb, 0x2b, 0x58, 0xdc, 0x27, 0x30, - 0x28, 0xe4, 0x88, 0x0d, 0x0b, 0x05, 0x92, 0x07, 0x60, 0xa9, 0x90, 0x24, 0xa1, 0x33, 0xeb, 0x4d, - 0x14, 0xc2, 0x44, 0xc7, 0x69, 0xaf, 0xfb, 0xc3, 0x80, 0xdd, 0x73, 0x94, 0x17, 0x3c, 0xb8, 0x21, - 0x1c, 0x99, 0xc0, 0xe0, 0x1b, 0x72, 0xe1, 0xb3, 0xf0, 0x8c, 0x85, 0x42, 0x72, 0xea, 0x87, 0xd2, - 0x6e, 0xc4, 0x61, 0x2f, 0xff, 0xf3, 0xfe, 0x74, 0x11, 0x07, 0x76, 0xb4, 0xd1, 0x36, 0x75, 0x54, - 0x6a, 0x38, 0x1d, 0x40, 0x5f, 0x1f, 0x3f, 0x60, 0x80, 0x0b, 0xc9, 0xb8, 0xfb, 0x09, 0xfa, 0x97, - 0xca, 0xe4, 0xa1, 0x60, 0x11, 0x5f, 0x20, 0xd9, 0x03, 0x33, 0xe2, 0xbe, 0xa6, 0x12, 0x1f, 0x09, - 0x81, 0x66, 0x18, 0xb3, 0x4b, 0x60, 0xbd, 0xe4, 0x4c, 0x46, 0xd0, 0x59, 0xb0, 0x50, 0x62, 0x28, - 0x3f, 0x5e, 0x6f, 0x50, 0x61, 0x79, 0x45, 0x93, 0xfb, 0xd3, 0x80, 0x5e, 0xfa, 0x56, 0x2d, 0xd3, - 0x09, 0xb4, 0xb9, 0x86, 0x11, 0xb6, 0x31, 0x32, 0xc7, 0x9d, 0xd9, 0x51, 0xaa, 0x54, 0x85, 0x86, - 0x97, 0x47, 0x16, 0xd4, 0x6d, 0xfc, 0x4d, 0x5d, 0x32, 0x86, 0xbe, 0x48, 0x1e, 0x86, 0xcb, 0xcb, - 0xa2, 0x06, 0x5e, 0xd5, 0xec, 0xfa, 0xb0, 0x7f, 0xc6, 0x91, 0x4a, 0xbc, 0x55, 0xef, 0xc9, 0x10, - 0x2c, 0x1a, 0xc9, 0x2b, 0xc6, 0xb5, 0x14, 0xfa, 0x46, 0x1c, 0x68, 0x05, 0x34, 0x5c, 0x45, 0x74, - 0x95, 0x2a, 0x91, 0xdd, 0xdd, 0x21, 0x1c, 0x94, 0xa1, 0x94, 0x16, 0xee, 0x09, 0xec, 0x3f, 0xc7, - 0x00, 0x6f, 0x49, 0x21, 0x2e, 0x57, 0x4e, 0xd3, 0xe5, 0x24, 0x1c, 0x5c, 0x6c, 0x02, 0x46, 0x97, - 0x99, 0x8e, 0x37, 0x7b, 0x92, 0x9d, 0xcf, 0x8b, 0x7a, 0x53, 0x7a, 0x25, 0x2e, 0xfc, 0xff, 0xc5, - 0x0f, 0x50, 0xd8, 0x66, 0xd2, 0xa8, 0x6e, 0x2a, 0xfa, 0x0b, 0x3f, 0x40, 0x4f, 0xb9, 0xdc, 0x23, - 0x38, 0xac, 0xa0, 0x6a, 0x3a, 0xef, 0x53, 0x9a, 0xff, 0x8a, 0x4e, 0x0c, 0x55, 0xa9, 0xa8, 0xa1, - 0xbe, 0x83, 0xa5, 0xb4, 0xc8, 0xe6, 0xd4, 0x28, 0xcc, 0xe9, 0x43, 0x68, 0xe9, 0x0a, 0xc2, 0x6e, - 0x24, 0x0f, 0xe9, 0x57, 0x27, 0x2e, 0x0b, 0x28, 0xf4, 0xd7, 0xdc, 0xda, 0xdf, 0x66, 0xa5, 0xbf, - 0xcf, 0x60, 0x47, 0x17, 0xaa, 0xc5, 0xcf, 0x54, 0x6c, 0x6c, 0x57, 0xf1, 0x35, 0x34, 0xe3, 0x6b, - 0x6d, 0xbe, 0x03, 0xad, 0xb5, 0xbf, 0xc6, 0x64, 0xc9, 0x94, 0x22, 0xd9, 0x3d, 0x8e, 0x5f, 0x52, - 0x49, 0x13, 0xb2, 0x5d, 0x2f, 0x39, 0xcf, 0x7e, 0x99, 0xd0, 0x4b, 0x47, 0x63, 0xe5, 0x0b, 0xc9, - 0xaf, 0xc9, 0x63, 0xb0, 0xd4, 0x1e, 0x92, 0xc3, 0x14, 0xbd, 0xf4, 0x0d, 0x72, 0x86, 0x55, 0xb3, - 0x5e, 0xd7, 0x57, 0xd0, 0x2d, 0x8e, 0x2e, 0xb9, 0x9b, 0xc6, 0xd5, 0xec, 0x8e, 0x73, 0xaf, 0xde, - 0xa9, 0x4b, 0x3d, 0x85, 0x76, 0xf6, 0xd5, 0x24, 0x76, 0x01, 0xaf, 0x5c, 0xe4, 0x4e, 0x8d, 0x27, - 0x27, 0x53, 0x1c, 0xfc, 0x9c, 0x4c, 0xcd, 0x16, 0xe5, 0x64, 0xea, 0x76, 0x85, 0xbc, 0x81, 0xdd, - 0xd2, 0xd4, 0x92, 0x2c, 0xbc, 0x6e, 0x85, 0x9c, 0xfb, 0x5b, 0xbc, 0x79, 0xb5, 0xd2, 0x60, 0x92, - 0x0a, 0xf8, 0xb6, 0x6a, 0xb5, 0xd3, 0x7c, 0x3a, 0xfa, 0x7c, 0xbc, 0xf2, 0xe5, 0x55, 0x34, 0x9f, - 0x2c, 0xd8, 0x7a, 0x1a, 0xd2, 0xa5, 0x1f, 0x50, 0x31, 0x8d, 0x53, 0xa6, 0x2a, 0x6f, 0x6e, 0x25, - 0xff, 0xb0, 0x47, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf5, 0x64, 0xe4, 0xee, 0xd1, 0x06, 0x00, - 0x00, + // 614 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6e, 0xd3, 0x4c, + 0x10, 0xfd, 0x5c, 0xe7, 0x73, 0x93, 0x69, 0x9a, 0x34, 0xdb, 0x36, 0x35, 0xe6, 0x47, 0x91, 0x85, + 0xaa, 0x48, 0x48, 0x89, 0x14, 0x54, 0x71, 0xc1, 0x0d, 0xb4, 0x88, 0x02, 0x02, 0x84, 0x0c, 0xad, + 0x04, 0x77, 0x4e, 0x3c, 0xa4, 0x96, 0x1c, 0x6f, 0xd8, 0x5d, 0x23, 0xf5, 0x82, 0x57, 0xe0, 0x9e, + 0x87, 0xe1, 0xdd, 0xd0, 0x7a, 0xd7, 0xbf, 0x38, 0x52, 0x2b, 0x71, 0xb7, 0x3b, 0x33, 0x3b, 0xe7, + 0xec, 0xd9, 0x33, 0x36, 0xc0, 0x8a, 0xfa, 0x6c, 0xb2, 0x66, 0x54, 0x50, 0x62, 0xc9, 0xf5, 0x7a, + 0xee, 0xce, 0x60, 0xef, 0x1c, 0xc5, 0x3b, 0x1a, 0x24, 0x11, 0x7a, 0xf8, 0x2d, 0x41, 0x2e, 0xc8, + 0x03, 0x59, 0x29, 0x03, 0xef, 0xfd, 0x15, 0xda, 0xc6, 0xc8, 0x18, 0x77, 0xbc, 0x52, 0xc4, 0x7d, + 0x0a, 0x83, 0xd2, 0x19, 0xbe, 0xa6, 0x31, 0x47, 0x72, 0x0c, 0x96, 0x2a, 0xb1, 0x8d, 0x91, 0x39, + 0xde, 0x99, 0xf5, 0x26, 0x0a, 0x61, 0xa2, 0xeb, 0x74, 0xd6, 0xfd, 0x69, 0xc0, 0xee, 0x39, 0x8a, + 0x0b, 0x16, 0xdd, 0x10, 0x8e, 0x4c, 0x60, 0xf0, 0x1d, 0x19, 0x0f, 0x69, 0x7c, 0x46, 0x63, 0x2e, + 0x98, 0x1f, 0xc6, 0xc2, 0xde, 0x92, 0x65, 0xaf, 0xfe, 0xf3, 0xfe, 0x4e, 0x11, 0x07, 0xb6, 0x75, + 0xd0, 0x36, 0x75, 0x55, 0x16, 0x38, 0x1d, 0x40, 0x5f, 0x2f, 0x3f, 0x62, 0x84, 0x0b, 0x41, 0x99, + 0xfb, 0x19, 0xfa, 0x97, 0x2a, 0xe4, 0x21, 0xa7, 0x09, 0x5b, 0x20, 0xd9, 0x03, 0x33, 0x61, 0xa1, + 0xa6, 0x22, 0x97, 0x84, 0x40, 0x2b, 0x96, 0xec, 0x52, 0x58, 0x2f, 0x5d, 0x93, 0x11, 0xec, 0x2c, + 0x68, 0x2c, 0x30, 0x16, 0x9f, 0xae, 0xd7, 0xa8, 0xb0, 0xbc, 0x72, 0xc8, 0xfd, 0x65, 0x40, 0x2f, + 0xbb, 0xab, 0x96, 0xe9, 0x04, 0x3a, 0x4c, 0xc3, 0x70, 0xad, 0xd4, 0x51, 0xa6, 0x54, 0x8d, 0x86, + 0x57, 0x54, 0x96, 0xd4, 0x95, 0x0c, 0x36, 0xaa, 0x4b, 0xc6, 0xd0, 0xe7, 0xe9, 0xc5, 0x30, 0xb8, + 0x2c, 0x6b, 0xe0, 0xd5, 0xc3, 0x6e, 0x08, 0xfb, 0x67, 0x0c, 0x7d, 0x81, 0xb7, 0x7a, 0x7b, 0x32, + 0x04, 0xcb, 0x4f, 0xc4, 0x15, 0x65, 0x5a, 0x0a, 0xbd, 0x23, 0x0e, 0xb4, 0x23, 0x3f, 0x5e, 0x26, + 0xfe, 0x32, 0x53, 0x22, 0xdf, 0xbb, 0x43, 0x38, 0xa8, 0x42, 0x29, 0x2d, 0xdc, 0x13, 0xd8, 0x7f, + 0x81, 0x11, 0xde, 0x92, 0x82, 0x6c, 0x57, 0x3d, 0xa6, 0xdb, 0x09, 0x38, 0xb8, 0x58, 0x47, 0xd4, + 0x0f, 0x72, 0x1d, 0x6f, 0x76, 0x25, 0xbb, 0xf0, 0x8b, 0xba, 0x53, 0xb6, 0x25, 0x2e, 0xfc, 0xff, + 0x35, 0x8c, 0x90, 0xdb, 0x66, 0xfa, 0x50, 0xdd, 0x4c, 0xf4, 0x97, 0x61, 0x84, 0x9e, 0x4a, 0xb9, + 0x47, 0x70, 0x58, 0x43, 0xd5, 0x74, 0x3e, 0x64, 0x34, 0xff, 0x15, 0x1d, 0x09, 0x55, 0xeb, 0xa8, + 0xa1, 0x7e, 0x80, 0xa5, 0xb4, 0xc8, 0x7d, 0x6a, 0x94, 0x7c, 0xfa, 0x08, 0xda, 0xba, 0x03, 0xb7, + 0xb7, 0xd2, 0x8b, 0xf4, 0xeb, 0x8e, 0xcb, 0x0b, 0x4a, 0xef, 0x6b, 0x6e, 0x7c, 0xdf, 0x56, 0xed, + 0x7d, 0x9f, 0xc3, 0xb6, 0x6e, 0xd4, 0x88, 0x9f, 0xab, 0xb8, 0xb5, 0x59, 0xc5, 0x37, 0xd0, 0x92, + 0xdb, 0xc6, 0xf3, 0x0e, 0xb4, 0x57, 0xe1, 0x0a, 0xd3, 0x21, 0x53, 0x8a, 0xe4, 0x7b, 0x59, 0x1f, + 0xf8, 0xc2, 0x4f, 0xc9, 0x76, 0xbd, 0x74, 0x3d, 0xfb, 0x6d, 0x42, 0x2f, 0xb3, 0xc6, 0x32, 0xe4, + 0x82, 0x5d, 0x93, 0x27, 0x60, 0xa9, 0x39, 0x24, 0x87, 0x19, 0x7a, 0xe5, 0x1b, 0xe4, 0x0c, 0xeb, + 0x61, 0x3d, 0xae, 0xaf, 0xa1, 0x5b, 0xb6, 0x2e, 0xb9, 0x9b, 0xd5, 0x35, 0xcc, 0x8e, 0x73, 0xaf, + 0x39, 0xa9, 0x5b, 0x3d, 0x83, 0x4e, 0xfe, 0xd5, 0x24, 0x76, 0x09, 0xaf, 0xda, 0xe4, 0x4e, 0x43, + 0xa6, 0x20, 0x53, 0x36, 0x7e, 0x41, 0xa6, 0x61, 0x8a, 0x0a, 0x32, 0x4d, 0xb3, 0x42, 0xde, 0xc2, + 0x6e, 0xc5, 0xb5, 0x24, 0x2f, 0x6f, 0x1a, 0x21, 0xe7, 0xfe, 0x86, 0x6c, 0xd1, 0xad, 0x62, 0x4c, + 0x52, 0x03, 0xdf, 0xd4, 0xad, 0xd1, 0xcd, 0xa7, 0xc7, 0x5f, 0x1e, 0x2e, 0x43, 0x71, 0x95, 0xcc, + 0x27, 0x0b, 0xba, 0x9a, 0x06, 0x54, 0x84, 0x71, 0x90, 0x70, 0xc1, 0x42, 0xe4, 0x53, 0x79, 0x70, + 0xaa, 0x4e, 0xcf, 0xad, 0xf4, 0x4f, 0xf6, 0xf8, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x41, 0xea, + 0x38, 0x29, 0xd7, 0x06, 0x00, 0x00, } diff --git a/rpc/get_module.go b/rpc/get_module.go index b913d9c..d146a8f 100644 --- a/rpc/get_module.go +++ b/rpc/get_module.go @@ -10,17 +10,25 @@ import ( ) func (s *Server) GetModule(ctx context.Context, request *moarpb.GetModuleRequest) (*moarpb.GetModuleResponse, error) { + var modules []*moarpb.Module + if request.ModuleName == "" { - return nil, twirp.RequiredArgumentError("moduleName") - } - module, err := s.registry.GetModule(ctx, request.ModuleName, false) - if errors.Is(err, registry.ModuleNotFound) { - return nil, twirp.NotFoundError("module not found: " + request.ModuleName) - } else if err != nil { - return nil, twirp.InternalErrorWith(err) + mods, err := s.registry.GetAllModules(ctx, false) + if err != nil { + return nil, twirp.InternalErrorWith(err) + } + for _, module := range mods { + modules = append(modules, moduleToDto(module)) + } + } else { + module, err := s.registry.GetModule(ctx, request.ModuleName, false) + if errors.Is(err, registry.ModuleNotFound) { + return nil, twirp.NotFoundError("module not found: " + request.ModuleName) + } else if err != nil { + return nil, twirp.InternalErrorWith(err) + } + modules = append(modules, moduleToDto(module)) } - mod := moduleToDto(module) - - return &moarpb.GetModuleResponse{Module: mod}, nil + return &moarpb.GetModuleResponse{Module: modules}, nil } diff --git a/rpc/server.go b/rpc/server.go index bc5a53c..92deef7 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -9,6 +9,7 @@ import ( ) type RegistryReader interface { + GetAllModules(ctx context.Context, loadData bool) ([]*internal.Module, error) GetModule(ctx context.Context, name string, loadData bool) (*internal.Module, error) Close() error }