diff --git a/api/api.go b/api/api.go index a7c564e..f064241 100644 --- a/api/api.go +++ b/api/api.go @@ -12,6 +12,7 @@ import ( "github.com/vocdoni/saas-backend/account" "github.com/vocdoni/saas-backend/db" "github.com/vocdoni/saas-backend/notifications" + "github.com/vocdoni/saas-backend/objectstorage" "github.com/vocdoni/saas-backend/stripe" "github.com/vocdoni/saas-backend/subscriptions" "go.vocdoni.io/dvote/apiclient" @@ -33,6 +34,7 @@ type APIConfig struct { Account *account.Account MailService notifications.NotificationService WebAppURL string + ServerURL string // FullTransparentMode if true allows signing all transactions and does not // modify any of them. FullTransparentMode bool @@ -40,6 +42,8 @@ type APIConfig struct { StripeClient *stripe.StripeClient // Subscriptions permissions manager Subscriptions *subscriptions.Subscriptions + // Object storage + ObjectStorage *objectstorage.ObjectStorageClient } // API type represents the API HTTP server with JWT authentication capabilities. @@ -54,9 +58,11 @@ type API struct { mail notifications.NotificationService secret string webAppURL string + serverURL string transparentMode bool stripe *stripe.StripeClient subscriptions *subscriptions.Subscriptions + objectStorage *objectstorage.ObjectStorageClient } // New creates a new API HTTP server. It does not start the server. Use Start() for that. @@ -74,9 +80,11 @@ func New(conf *APIConfig) *API { mail: conf.MailService, secret: conf.Secret, webAppURL: conf.WebAppURL, + serverURL: conf.ServerURL, transparentMode: conf.FullTransparentMode, stripe: conf.StripeClient, subscriptions: conf.Subscriptions, + objectStorage: conf.ObjectStorage, } } @@ -162,6 +170,9 @@ func (a *API) initRouter() http.Handler { // get stripe subscription portal session info log.Infow("new route", "method", "GET", "path", subscriptionsPortal) r.Get(subscriptionsPortal, a.createSubscriptionPortalSessionHandler) + // upload an image to the object storage + log.Infow("new route", "method", "POST", "path", objectStorageUploadTypedEndpoint) + r.Post(objectStorageUploadTypedEndpoint, a.uploadImageWithFormHandler) }) // Public routes @@ -213,6 +224,9 @@ func (a *API) initRouter() http.Handler { // handle stripe webhook log.Infow("new route", "method", "POST", "path", subscriptionsWebhook) r.Post(subscriptionsWebhook, a.handleWebhook) + // upload an image to the object storage + log.Infow("new route", "method", "GET", "path", objectStorageDownloadTypedEndpoint) + r.Get(objectStorageDownloadTypedEndpoint, a.downloadImageInlineHandler) }) a.router = r return r diff --git a/api/docs.md b/api/docs.md index 1b8dac3..4dbfdf7 100644 --- a/api/docs.md +++ b/api/docs.md @@ -39,6 +39,10 @@ - [🛒 Create Checkout session](#-create-checkout-session) - [🛍️ Get Checkout session info](#-get-checkout-session-info) - [🔗 Create Subscription Portal Session](#-create-subscription-portal-session) +- [📦 Storage](#-storage) + - [ 🌄 Upload image](#-upload-image) + - [ 📄 Get object](#-get-object) + @@ -994,4 +998,53 @@ This request can be made only by organization admins. |:---:|:---:|:---| | `401` | `40001` | `user not authorized` | | `400` | `40011` | `no organization provided` | -| `500` | `50002` | `internal server error` | \ No newline at end of file +| `500` | `50002` | `internal server error` | + +## 📦 Storage + +### 🌄 Upload image + +* **Path** `/storage` +* **Method** `POST` + +Accepting files uploaded by forms as such: +```html +
+

+

+

+

+

+``` + +* **Response** + +```json +{ + "urls": ["https://file1.store.com","https://file1.store.com"] +} +``` + +* **Errors** + +| HTTP Status | Error code | Message | +|:---:|:---:|:---| +| `401` | `40001` | `user not authorized` | +| `400` | `40024` | `the obejct/parameters provided are invalid` | +| `500` | `50002` | `internal server error` | +| `500` | `50006` | `internal storage error` | + + +### 📄 Get object +This method return if exists, in inline mode. the image/file of the provided by the obectID + +* **Path** `/storage/{objectID}` +* **Method** `GET` + +* **Errors** + +| HTTP Status | Error code | Message | +|:---:|:---:|:---| +| `400` | `40024` | `the obejct/parameters provided are invalid` | +| `500` | `50002` | `internal server error` | +| `500` | `50006` | `internal storage error` | \ No newline at end of file diff --git a/api/errors_definition.go b/api/errors_definition.go index 10747e2..8fb8522 100644 --- a/api/errors_definition.go +++ b/api/errors_definition.go @@ -49,10 +49,12 @@ var ( ErrOganizationSubscriptionIncative = Error{Code: 40021, HTTPstatus: http.StatusBadRequest, Err: fmt.Errorf("organization subscription not active")} ErrNoDefaultPLan = Error{Code: 40022, HTTPstatus: http.StatusBadRequest, Err: fmt.Errorf("did not found default plan for organization")} ErPlanNotFound = Error{Code: 40023, HTTPstatus: http.StatusNotFound, Err: fmt.Errorf("plan not found")} + ErrStorageInvalidObject = Error{Code: 40024, HTTPstatus: http.StatusBadRequest, Err: fmt.Errorf("the obejct/parameters provided are invalid")} ErrMarshalingServerJSONFailed = Error{Code: 50001, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("marshaling (server-side) JSON failed")} ErrGenericInternalServerError = Error{Code: 50002, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("internal server error")} ErrCouldNotCreateFaucetPackage = Error{Code: 50003, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("could not create faucet package")} ErrVochainRequestFailed = Error{Code: 50004, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("vochain request failed")} ErrStripeError = Error{Code: 50005, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("stripe error")} + ErrInternalStorageError = Error{Code: 50006, HTTPstatus: http.StatusInternalServerError, Err: fmt.Errorf("internal storage error")} ) diff --git a/api/object_storage.go b/api/object_storage.go new file mode 100644 index 0000000..0690ff2 --- /dev/null +++ b/api/object_storage.go @@ -0,0 +1,104 @@ +package api + +import ( + "fmt" + "net/http" + "regexp" + + "github.com/go-chi/chi/v5" +) + +// isObjectNameRgx is a regular expression to match object names. +var isObjectNameRgx = regexp.MustCompile(`^([a-zA-Z0-9]+)\.(jpg|jpeg|png)`) + +// uploadImageWithFormHandler handles the uploading of images through a multipart form. +// It expects the request to contain a "file" field with one or more files to be uploaded. +func (a *API) uploadImageWithFormHandler(w http.ResponseWriter, r *http.Request) { + // check if the user is authenticated + // get the user from the request context + user, ok := userFromContext(r.Context()) + if !ok { + ErrUnauthorized.Write(w) + return + } + + // 32 MB is the default used by FormFile() function + if err := r.ParseMultipartForm(32 << 20); err != nil { + ErrStorageInvalidObject.With("could not parse form").Write(w) + return + } + + // Get a reference to the fileHeaders. + // They are accessible only after ParseMultipartForm is called + files := r.MultipartForm.File["file"] + var returnURLs []string + for _, fileHeader := range files { + // Open the file + file, err := fileHeader.Open() + if err != nil { + ErrStorageInvalidObject.Withf("cannot open file %s", err.Error()).Write(w) + break + } + defer func() { + if err := file.Close(); err != nil { + ErrStorageInvalidObject.Withf("cannot close file %s", err.Error()).Write(w) + return + } + }() + // upload the file using the object storage client + // and get the URL of the uploaded file + storedFileID, err := a.objectStorage.Put(file, fileHeader.Size, user.Email) + if err != nil { + ErrInternalStorageError.With(err.Error()).Write(w) + break + } + returnURLs = append(returnURLs, objectURL(a.serverURL, storedFileID)) + } + httpWriteJSON(w, map[string][]string{"urls": returnURLs}) +} + +// downloadImageInlineHandler handles the HTTP request to download an image inline. +// It retrieves the object ID from the URL parameters, fetches the object from the +// object storage, and writes the object data to the HTTP response with appropriate +// headers for inline display. +func (a *API) downloadImageInlineHandler(w http.ResponseWriter, r *http.Request) { + objectName := chi.URLParam(r, "objectName") + if objectName == "" { + ErrMalformedURLParam.With("objectName is required").Write(w) + return + } + objectID, ok := objectIDfromName(objectName) + if !ok { + ErrStorageInvalidObject.With("invalid objectName").Write(w) + return + } + // get the object from the object storage client + object, err := a.objectStorage.Get(objectID) + if err != nil { + ErrStorageInvalidObject.Withf("cannot get object %s", err.Error()).Write(w) + return + } + // write the object to the response + w.Header().Set("Content-Type", object.ContentType) + // w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.Header().Set("Content-Disposition", "inline") + if _, err := w.Write(object.Data); err != nil { + ErrInternalStorageError.Withf("cannot write object %s", err.Error()).Write(w) + return + } +} + +// objectURL returns the URL for the object with the given objectID. +func objectURL(baseURL, objectID string) string { + return fmt.Sprintf("%s/storage/%s", baseURL, objectID) +} + +// objectIDfromURL returns the objectID from the given URL. If the URL is not an +// object URL, it returns an empty string and false. +func objectIDfromName(url string) (string, bool) { + objectID := isObjectNameRgx.FindStringSubmatch(url) + if len(objectID) != 3 { + return "", false + } + return objectID[1], true +} diff --git a/api/plans.go b/api/plans.go index 6b5c193..d000a63 100644 --- a/api/plans.go +++ b/api/plans.go @@ -3,6 +3,8 @@ package api import ( "net/http" "strconv" + + "github.com/go-chi/chi/v5" ) // getSubscriptionsHandler handles the request to get the subscriptions of an organization. @@ -20,7 +22,7 @@ func (a *API) getPlansHandler(w http.ResponseWriter, r *http.Request) { func (a *API) planInfoHandler(w http.ResponseWriter, r *http.Request) { // get the plan ID from the URL - planID := r.URL.Query().Get("planID") + planID := chi.URLParam(r, "planID") // check the the planID is not empty if planID == "" { ErrMalformedURLParam.Withf("planID is required").Write(w) diff --git a/api/routes.go b/api/routes.go index 20c3a14..2e95e81 100644 --- a/api/routes.go +++ b/api/routes.go @@ -72,4 +72,9 @@ const ( subscriptionsCheckoutSession = "/subscriptions/checkout/{sessionID}" // GET /subscriptions/portal to get the stripe subscription portal URL subscriptionsPortal = "/subscriptions/{address}/portal" + // object storage routes + // POST /storage/{origin} to upload an image to the object storage + objectStorageUploadTypedEndpoint = "/storage" + // GET /storage/{origin}/{filename} to download an image from the object storage + objectStorageDownloadTypedEndpoint = "/storage/{objectName}" ) diff --git a/api/users.go b/api/users.go index d4b9fee..87034c2 100644 --- a/api/users.go +++ b/api/users.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/go-chi/chi/v5" "github.com/vocdoni/saas-backend/db" "github.com/vocdoni/saas-backend/internal" "github.com/vocdoni/saas-backend/notifications/mailtemplates" @@ -174,7 +175,7 @@ func (a *API) verifyUserAccountHandler(w http.ResponseWriter, r *http.Request) { // returned. func (a *API) userVerificationCodeInfoHandler(w http.ResponseWriter, r *http.Request) { // get the user email of the user from the request query - userEmail := r.URL.Query().Get("email") + userEmail := chi.URLParam(r, "email") // check the email is not empty if userEmail == "" { ErrInvalidUserData.With("no email provided").Write(w) diff --git a/cmd/service/main.go b/cmd/service/main.go index 9192549..1a8b5a5 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -12,6 +12,7 @@ import ( "github.com/vocdoni/saas-backend/db" "github.com/vocdoni/saas-backend/notifications/mailtemplates" "github.com/vocdoni/saas-backend/notifications/smtp" + "github.com/vocdoni/saas-backend/objectstorage" "github.com/vocdoni/saas-backend/stripe" "github.com/vocdoni/saas-backend/subscriptions" "go.vocdoni.io/dvote/apiclient" @@ -20,6 +21,7 @@ import ( func main() { // define flags + flag.String("server", "http://localhost:8080", "The full URL of the server (http or https)") flag.StringP("host", "h", "0.0.0.0", "listen address") flag.IntP("port", "p", 8080, "listen port") flag.StringP("secret", "s", "", "API secret") @@ -46,6 +48,7 @@ func main() { } viper.AutomaticEnv() // read the configuration + server := viper.GetString("server") host := viper.GetString("host") port := viper.GetInt("port") apiEndpoint := viper.GetString("vocdoniApi") @@ -113,6 +116,7 @@ func main() { Client: apiClient, Account: acc, WebAppURL: webURL, + ServerURL: server, FullTransparentMode: fullTransparentMode, StripeClient: stripeClient, } @@ -144,6 +148,10 @@ func main() { DB: database, }) apiConf.Subscriptions = subscriptions + // initialize the s3 like object storage + apiConf.ObjectStorage = objectstorage.New(&objectstorage.ObjectStorageConfig{ + DB: database, + }) // create the local API server api.New(apiConf).Start() log.Infow("server started", "host", host, "port", port) diff --git a/db/helpers.go b/db/helpers.go index 9f55eb3..3972686 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -96,6 +96,10 @@ func (ms *MongoStorage) initCollections(database string) error { if ms.plans, err = getCollection("plans"); err != nil { return err } + // objects collection + if ms.objects, err = getCollection("objects"); err != nil { + return err + } return nil } diff --git a/db/mongo.go b/db/mongo.go index 89fc6c8..3d52066 100644 --- a/db/mongo.go +++ b/db/mongo.go @@ -27,6 +27,7 @@ type MongoStorage struct { organizations *mongo.Collection organizationInvites *mongo.Collection plans *mongo.Collection + objects *mongo.Collection } type Options struct { @@ -119,6 +120,10 @@ func (ms *MongoStorage) Reset() error { if err := ms.plans.Drop(ctx); err != nil { return err } + // drop the objects collection + if err := ms.objects.Drop(ctx); err != nil { + return err + } // init the collections if err := ms.initCollections(ms.database); err != nil { return err diff --git a/db/object.go b/db/object.go new file mode 100644 index 0000000..7a3d04d --- /dev/null +++ b/db/object.go @@ -0,0 +1,72 @@ +package db + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// The Object entity represents a generic object stored in the database +// intended for s3-like storage. + +// Object retrieves an object from the MongoDB collection by its ID. +func (ms *MongoStorage) Object(id string) (*Object, error) { + ms.keysLock.RLock() + defer ms.keysLock.RUnlock() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // find the object in the database + result := ms.objects.FindOne(ctx, bson.M{"_id": id}) + obj := &Object{} + if err := result.Decode(obj); err != nil { + if err == mongo.ErrNoDocuments { + return nil, ErrNotFound + } + return nil, err + } + return obj, nil +} + +// SetObject sets the object data for the given objectID. If the +// object does not exist, it will be created with the given data, otherwise it +// will be updated. +func (ms *MongoStorage) SetObject(objectID, userID, contentType string, data []byte) error { + object := &Object{ + ID: objectID, + Data: data, + CreatedAt: time.Now(), + UserID: userID, + ContentType: contentType, + } + ms.keysLock.Lock() + defer ms.keysLock.Unlock() + return ms.setObject(object) +} + +// RemoveObject removes the object data for the given objectID. +func (ms *MongoStorage) RemoveObject(objectID string) error { + ms.keysLock.Lock() + defer ms.keysLock.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := ms.objects.DeleteOne(ctx, bson.M{"_id": objectID}) + return err +} + +func (ms *MongoStorage) setObject(object *Object) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + opts := options.ReplaceOptions{} + opts.Upsert = new(bool) + *opts.Upsert = true + _, err := ms.objects.ReplaceOne(ctx, bson.M{"_id": object.ID}, object, &opts) + if err != nil { + return fmt.Errorf("cannot update object: %w", err) + } + return err +} diff --git a/db/types.go b/db/types.go index 1f0dd16..7ff2877 100644 --- a/db/types.go +++ b/db/types.go @@ -134,3 +134,14 @@ type OrganizationInvite struct { Role UserRole `json:"role" bson:"role"` Expiration time.Time `json:"expiration" bson:"expiration"` } + +// Object represents a user uploaded object Includes user defined ID and the data +// as a byte array. +type Object struct { + ID string `json:"id" bson:"_id"` + Name string `json:"name" bson:"name"` + Data []byte `json:"data" bson:"data"` + CreatedAt time.Time `json:"createdAt" bson:"createdAt"` + UserID string `json:"userId" bson:"userId"` + ContentType string `json:"contentType" bson:"contentType"` +} diff --git a/example.env b/example.env index 2c9e1e1..ad1fb28 100644 --- a/example.env +++ b/example.env @@ -1,3 +1,4 @@ +VOCDONI_SERVERURL=http://localhost:8080 VOCDONI_PORT=8080 VOCDONI_SECRET=supersecret VOCDONI_PRIVATEKEY=vochain-private-key @@ -6,5 +7,6 @@ VOCDONI_SMTPUSERNAME=admin VOCDONI_SMTPPASSWORD=password VOCDONI_EMAILFROMADDRESS=admin@email.com VOCDONI_EMAILFROMADDRESS=admin@email.com -STRIPE_API_SECRET=stripe_key -STRIPE_WEBHOOK_SECRET=stripe_webhook_key +VOCDONI_STRIPEAPISECRET=test +VOCDONI_STRIPEWEBHOOKSEC=test +VOCDONI_WEBURL=test diff --git a/objectstorage/objectstorage.go b/objectstorage/objectstorage.go new file mode 100644 index 0000000..0b04c06 --- /dev/null +++ b/objectstorage/objectstorage.go @@ -0,0 +1,121 @@ +package objectstorage + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + + "github.com/vocdoni/saas-backend/db" +) + +var ( + ErrorObjectNotFound = fmt.Errorf("object not found") + ErrorInvalidObjectID = fmt.Errorf("invalid object ID") + ErrorFileTypeNotSupported = fmt.Errorf("file type not supported") +) + +type ObjectFileType string + +const ( + FileTypeJPEG ObjectFileType = "image/jpeg" + FileTypePNG ObjectFileType = "image/png" + FileTypeJPG ObjectFileType = "image/jpg" +) + +var DefaultSupportedFileTypes = map[ObjectFileType]bool{ + FileTypeJPEG: true, + FileTypePNG: true, + FileTypeJPG: true, +} + +type ObjectStorageConfig struct { + DB *db.MongoStorage + SupportedTypes []ObjectFileType +} + +type ObjectStorageClient struct { + db *db.MongoStorage + supportedTypes map[ObjectFileType]bool +} + +// New initializes a new ObjectStorageClient with the provided API credentials and configuration. +// It sets up a MinIO client and verifies the existence of the specified bucket. +func New(conf *ObjectStorageConfig) *ObjectStorageClient { + if conf == nil { + return nil + } + supportedTypes := DefaultSupportedFileTypes + for _, t := range conf.SupportedTypes { + supportedTypes[t] = true + } + return &ObjectStorageClient{ + db: conf.DB, + supportedTypes: supportedTypes, + } +} + +// key is set in a string and can have a directory like notation (for example "folder-path/hello-world.txt") +func (osc *ObjectStorageClient) Get(objectID string) (*db.Object, error) { + if objectID == "" { + return nil, ErrorInvalidObjectID + } + + object, err := osc.db.Object(objectID) + if err != nil { + if err == db.ErrNotFound { + return nil, ErrorObjectNotFound + } + return nil, fmt.Errorf("error retrieving object: %w", err) + } + + return object, nil +} + +// uploadObject uploads the object image with the given objectID, associated to +// the user with the given userFID and the community with the given communityID. +// If the objectID is empty, it calculates the objectID from the data. It returns +// the URL of the uploaded object image. It stores the object in the database. +// If an error occurs, it returns an empty string and the error. +func (osc *ObjectStorageClient) Put(data io.Reader, size int64, userID string) (string, error) { + // Create a buffer of the appropriate size + buff := make([]byte, size) + _, err := data.Read(buff) + if err != nil { + return "", fmt.Errorf("cannot read file %s", err.Error()) + } + // checking the content type + // so we don't allow files other than images + filetype := http.DetectContentType(buff) + // extract type/extesion from the filetype + fileExtension := strings.Split(filetype, "/")[1] + + if !osc.supportedTypes[ObjectFileType(fileExtension)] { + return "ObjectFileType", ErrorFileTypeNotSupported + } + + objectID, err := calculateObjectID(buff) + if err != nil { + return "", fmt.Errorf("error calculating objectID: %w", err) + } + // store the object in the database + if err := osc.db.SetObject(objectID, userID, filetype, buff); err != nil { + return "", fmt.Errorf("cannot set object: %w", err) + } + // return objectURL(osc.serverURL, objectID, fileExtension), nil + return fmt.Sprintf("%s.%s", objectID, fileExtension), nil +} + +// calculateObjectID calculates the objectID from the given data. The objectID +// is the first 12 bytes of the md5 hash of the data. If an error occurs, it +// returns an empty string and the error. +func calculateObjectID(data []byte) (string, error) { + md5hash := md5.New() + if _, err := md5hash.Write(data); err != nil { + return "", fmt.Errorf("cannot calculate hash: %w", err) + } + bhash := md5hash.Sum(nil)[:12] + return hex.EncodeToString(bhash), nil +}