From ab80ef0df6f2435bbfa52a20a48b8a27ac6b9ed2 Mon Sep 17 00:00:00 2001 From: Eric Bower Date: Sat, 18 Jan 2025 09:27:41 -0500 Subject: [PATCH] chore(prose): migrate images to pgs --- cmd/scripts/prose-imgs-migrate/main.go | 99 +++++++++++++++ filehandlers/imgs/handler.go | 12 +- filehandlers/imgs/img.go | 26 +--- imgs/api.go | 168 ------------------------- imgs/html/rss.page.tmpl | 1 - imgs/public/.gitkeep | 0 pgs/web.go | 2 - prose/api.go | 58 ++++----- 8 files changed, 135 insertions(+), 231 deletions(-) create mode 100644 cmd/scripts/prose-imgs-migrate/main.go delete mode 100644 imgs/api.go delete mode 100644 imgs/html/rss.page.tmpl delete mode 100644 imgs/public/.gitkeep diff --git a/cmd/scripts/prose-imgs-migrate/main.go b/cmd/scripts/prose-imgs-migrate/main.go new file mode 100644 index 00000000..e8781e1b --- /dev/null +++ b/cmd/scripts/prose-imgs-migrate/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "bytes" + "io" + "log/slog" + "path/filepath" + "time" + + "github.com/picosh/pico/db" + "github.com/picosh/pico/db/postgres" + "github.com/picosh/pico/prose" + "github.com/picosh/pico/shared" + "github.com/picosh/pico/shared/storage" + sst "github.com/picosh/pobj/storage" + sendUtils "github.com/picosh/send/utils" +) + +func bail(err error) { + if err != nil { + panic(err) + } +} + +func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error { + toSite := filepath.Join("prose", fpath) + logger.Info("uploading object", "bucket", bucket.Name, "object", toSite) + buf := &bytes.Buffer{} + size, err := io.Copy(buf, rdr) + if err != nil { + return err + } + + _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{ + Mtime: time.Now().Unix(), + Size: size, + }) + return err +} + +func images(logger *slog.Logger, dbh db.DB, st storage.StorageServe, bucket sst.Bucket, user *db.User) error { + posts, err := dbh.FindPostsForUser(&db.Pager{Num: 2000, Page: 0}, user.ID, "imgs") + if err != nil { + return err + } + + if len(posts.Data) == 0 { + logger.Info("user does not have any images, skipping") + return nil + } + + imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID)) + if err != nil { + logger.Info("user does not have an images dir, skipping") + return nil + } + + /* imgs, err := st.ListObjects(imgBucket, "/", false) + if err != nil { + return err + } */ + + for _, posts := range posts.Data { + rdr, _, err := st.GetObject(imgBucket, posts.Filename) + if err != nil { + logger.Error("get object", "err", err) + return err + } + err = upload(logger, st, bucket, posts.Filename, rdr) + if err != nil { + return err + } + } + + return nil +} + +func main() { + cfg := prose.NewConfigSite() + logger := cfg.Logger + picoDb := postgres.NewDB(cfg.DbURL, logger) + st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass) + bail(err) + + users, err := picoDb.FindUsers() + bail(err) + + for _, user := range users { + logger.Info("migrating user images", "user", user.Name) + + bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID)) + bail(err) + _, _ = picoDb.InsertProject(user.ID, "prose", "prose") + err = images(logger, picoDb, st, bucket, user) + if err != nil { + logger.Error("image uploader", "err", err) + } + } +} diff --git a/filehandlers/imgs/handler.go b/filehandlers/imgs/handler.go index 3964090e..53240094 100644 --- a/filehandlers/imgs/handler.go +++ b/filehandlers/imgs/handler.go @@ -47,6 +47,10 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S } } +func (h *UploadImgHandler) getObjectPath(fpath string) string { + return filepath.Join("prose", fpath) +} + func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) { user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"]) if err != nil { @@ -71,12 +75,12 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F FModTime: *post.UpdatedAt, } - bucket, err := h.Storage.GetBucket(user.ID) + bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID)) if err != nil { return nil, nil, err } - contents, _, err := h.Storage.GetObject(bucket, post.Filename) + contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename)) if err != nil { return nil, nil, err } @@ -218,13 +222,13 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err return fmt.Errorf("error for %s: %v", filename, err) } - bucket, err := h.Storage.UpsertBucket(user.ID) + bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID)) if err != nil { return err } logger.Info("deleting image") - err = h.Storage.DeleteObject(bucket, filename) + err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename)) if err != nil { return err } diff --git a/filehandlers/imgs/img.go b/filehandlers/imgs/img.go index 23cc0aaa..83b296a6 100644 --- a/filehandlers/imgs/img.go +++ b/filehandlers/imgs/img.go @@ -49,7 +49,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error { return nil } - bucket, err := h.Storage.UpsertBucket(data.User.ID) + bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID)) if err != nil { return err } @@ -58,7 +58,7 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error { fname, _, err := h.Storage.PutObject( bucket, - data.Filename, + h.getObjectPath(data.Filename), sendutils.NopReaderAtCloser(reader), &sendutils.FileEntry{}, ) @@ -128,18 +128,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error { logger.Error("post could not create", "err", err.Error()) return fmt.Errorf("error for %s: %v", data.Filename, err) } - - if len(data.Tags) > 0 { - logger.Info( - "found post tags, replacing with old tags", - "tags", strings.Join(data.Tags, ","), - ) - err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Post.ID) - if err != nil { - logger.Error("post could not replace tags", "err", err.Error()) - return fmt.Errorf("error for %s: %v", data.Filename, err) - } - } } else { if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) { logger.Info("image found, but image is identical, skipping") @@ -167,16 +155,6 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error { logger.Error("post could not update", "err", err.Error()) return fmt.Errorf("error for %s: %v", data.Filename, err) } - - logger.Info( - "found post tags, replacing with old tags", - "tags", strings.Join(data.Tags, ","), - ) - err = h.DBPool.ReplaceTagsForPost(data.Tags, data.Cur.ID) - if err != nil { - logger.Error("post could not replace tags", "err", err.Error()) - return fmt.Errorf("error for %s: %v", data.Filename, err) - } } return nil diff --git a/imgs/api.go b/imgs/api.go deleted file mode 100644 index 0d1ab104..00000000 --- a/imgs/api.go +++ /dev/null @@ -1,168 +0,0 @@ -package imgs - -import ( - "fmt" - "html/template" - "net/http" - "net/url" - "path/filepath" - - "github.com/picosh/pico/db" - "github.com/picosh/pico/pgs" - "github.com/picosh/pico/shared" - "github.com/picosh/pico/shared/storage" - "github.com/picosh/utils" -) - -type PostPageData struct { - ImgURL template.URL -} - -type BlogPageData struct { - Site *shared.SitePageData - PageTitle string - URL template.URL - Username string - Posts []template.URL -} - -var Space = "imgs" - -func ImgsListHandler(w http.ResponseWriter, r *http.Request) { - username := shared.GetUsernameFromRequest(r) - dbpool := shared.GetDB(r) - logger := shared.GetLogger(r) - cfg := shared.GetCfg(r) - - user, err := dbpool.FindUserForName(username) - if err != nil { - logger.Info("blog not found", "username", username) - http.Error(w, "blog not found", http.StatusNotFound) - return - } - - var posts []*db.Post - pager := &db.Pager{Num: 1000, Page: 0} - p, err := dbpool.FindPostsForUser(pager, user.ID, Space) - posts = p.Data - - if err != nil { - logger.Error(err.Error()) - http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError) - return - } - - ts, err := shared.RenderTemplate(cfg, []string{ - cfg.StaticPath("html/imgs.page.tmpl"), - }) - - if err != nil { - logger.Error(err.Error()) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - curl := shared.CreateURLFromRequest(cfg, r) - postCollection := make([]template.URL, 0, len(posts)) - for _, post := range posts { - url := cfg.ImgURL(curl, post.Username, post.Slug) - postCollection = append(postCollection, template.URL(url)) - } - - data := BlogPageData{ - Site: cfg.GetSiteData(), - PageTitle: fmt.Sprintf("%s imgs", username), - URL: template.URL(cfg.FullBlogURL(curl, username)), - Username: username, - Posts: postCollection, - } - - err = ts.Execute(w, data) - if err != nil { - logger.Error(err.Error()) - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -func anyPerm(proj *db.Project) bool { - return true -} - -func ImgRequest(w http.ResponseWriter, r *http.Request) { - subdomain := shared.GetSubdomain(r) - cfg := shared.GetCfg(r) - st := shared.GetStorage(r) - dbpool := shared.GetDB(r) - logger := shared.GetLogger(r) - username := shared.GetUsernameFromRequest(r) - - user, err := dbpool.FindUserForName(username) - if err != nil { - logger.Info("user not found", "user", username) - http.Error(w, "user not found", http.StatusNotFound) - return - } - - var imgOpts string - var slug string - if !cfg.IsSubdomains() || subdomain == "" { - slug, _ = url.PathUnescape(shared.GetField(r, 1)) - imgOpts, _ = url.PathUnescape(shared.GetField(r, 2)) - } else { - slug, _ = url.PathUnescape(shared.GetField(r, 0)) - imgOpts, _ = url.PathUnescape(shared.GetField(r, 1)) - } - - opts, err := storage.UriToImgProcessOpts(imgOpts) - if err != nil { - errMsg := fmt.Sprintf("error processing img options: %s", err.Error()) - logger.Info(errMsg) - http.Error(w, errMsg, http.StatusUnprocessableEntity) - return - } - - // set default quality for web optimization - if opts.Quality == 0 { - opts.Quality = 80 - } - - ext := filepath.Ext(slug) - // set default format to be webp - if opts.Ext == "" && ext == "" { - opts.Ext = "webp" - } - - // Files can contain periods. `filepath.Ext` is greedy and will clip the last period in the slug - // and call that a file extension so we want to be explicit about what - // file extensions we clip here - for _, fext := range cfg.AllowedExt { - if ext == fext { - // users might add the file extension when requesting an image - // but we want to remove that - slug = utils.SanitizeFileExt(slug) - break - } - } - - post, err := FindImgPost(r, user, slug) - if err != nil { - errMsg := fmt.Sprintf("image not found %s/%s", user.Name, slug) - logger.Info(errMsg) - http.Error(w, errMsg, http.StatusNotFound) - return - } - - fname := post.Filename - router := pgs.NewWebRouter( - cfg, - logger, - dbpool, - st, - ) - router.ServeAsset(fname, opts, true, anyPerm, w, r) -} - -func FindImgPost(r *http.Request, user *db.User, slug string) (*db.Post, error) { - dbpool := shared.GetDB(r) - return dbpool.FindPostWithSlug(slug, user.ID, Space) -} diff --git a/imgs/html/rss.page.tmpl b/imgs/html/rss.page.tmpl deleted file mode 100644 index 3be5b53b..00000000 --- a/imgs/html/rss.page.tmpl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/imgs/public/.gitkeep b/imgs/public/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/pgs/web.go b/pgs/web.go index cb665d40..a91261e0 100644 --- a/pgs/web.go +++ b/pgs/web.go @@ -397,7 +397,6 @@ var imgRegex = regexp.MustCompile("(.+.(?:jpg|jpeg|png|gif|webp|svg))(/.+)") func (web *WebRouter) AssetRequest(w http.ResponseWriter, r *http.Request) { fname := r.PathValue("fname") if imgRegex.MatchString(fname) { - fmt.Println("HIT") web.ImageRequest(w, r) return } @@ -415,7 +414,6 @@ func (web *WebRouter) ImageRequest(w http.ResponseWriter, r *http.Request) { if len(matches) >= 3 { imgOpts = matches[2] } - fmt.Println("ZZZ", fname, imgOpts) opts, err := storage.UriToImgProcessOpts(imgOpts) if err != nil { diff --git a/prose/api.go b/prose/api.go index 8c867ae9..bf6b7382 100644 --- a/prose/api.go +++ b/prose/api.go @@ -5,6 +5,7 @@ import ( "fmt" "html/template" "net/http" + "net/http/httputil" "net/url" "os" "strconv" @@ -16,7 +17,6 @@ import ( "github.com/gorilla/feeds" "github.com/picosh/pico/db" "github.com/picosh/pico/db/postgres" - "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" "github.com/picosh/utils" @@ -170,13 +170,6 @@ func blogHandler(w http.ResponseWriter, r *http.Request) { } posts = p.Data - byUpdated := strings.Contains(r.URL.Path, "live") - if byUpdated { - slices.SortFunc(posts, func(a *db.Post, b *db.Post) int { - return b.UpdatedAt.Compare(*a.UpdatedAt) - }) - } - if err != nil { logger.Error(err.Error()) http.Error(w, "could not fetch posts for blog", http.StatusInternalServerError) @@ -438,14 +431,6 @@ func postHandler(w http.ResponseWriter, r *http.Request) { WithStyles: withStyles, } } else { - // TODO: HACK to support imgs slugs inside prose - // We definitely want to kill this feature in time - imgPost, err := imgs.FindImgPost(r, user, slug) - if err == nil && imgPost != nil { - imgs.ImgRequest(w, r) - return - } - notFound, err := dbpool.FindPostWithFilename("_404.md", user.ID, cfg.Space) contents := template.HTML("Oops! we can't seem to find this post.") title := "Post not found" @@ -645,13 +630,6 @@ func rssBlogHandler(w http.ResponseWriter, r *http.Request) { curl := shared.CreateURLFromRequest(cfg, r) blogUrl := cfg.FullBlogURL(curl, username) - byUpdated := strings.Contains(r.URL.Path, "live") - if byUpdated { - slices.SortFunc(posts, func(a *db.Post, b *db.Post) int { - return b.UpdatedAt.Compare(*a.UpdatedAt) - }) - } - feed := &feeds.Feed{ Id: blogUrl, Title: headerTxt.Title, @@ -692,10 +670,6 @@ func rssBlogHandler(w http.ResponseWriter, r *http.Request) { realUrl := cfg.FullPostURL(curl, post.Username, post.Slug) feedId := realUrl - if byUpdated { - feedId = fmt.Sprintf("%s:%s", realUrl, post.UpdatedAt.Format(time.RFC3339)) - } - item := &feeds.Item{ Id: feedId, Title: utils.FilenameToTitle(post.Filename, post.Title), @@ -859,13 +833,32 @@ func createMainRoutes(staticRoutes []shared.Route) []shared.Route { return routes } +func imgRequest(w http.ResponseWriter, r *http.Request) { + logger := shared.GetLogger(r) + username := shared.GetUsernameFromRequest(r) + destUrl, err := url.Parse(fmt.Sprintf("https://%s-prose.pgs.sh%s", username, r.URL.Path)) + if err != nil { + logger.Error("could not parse image proxy url", "username", username) + http.Error(w, "could not parse image proxy url", http.StatusInternalServerError) + return + } + logger.Info("proxy image request", "url", destUrl.String()) + + proxy := httputil.NewSingleHostReverseProxy(destUrl) + oldDirector := proxy.Director + proxy.Director = func(r *http.Request) { + oldDirector(r) + r.Host = destUrl.Host + r.URL = destUrl + } + proxy.ServeHTTP(w, r) +} + func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route { routes := []shared.Route{ shared.NewRoute("GET", "/", blogHandler), - shared.NewRoute("GET", "/live", blogHandler), shared.NewRoute("GET", "/_styles.css", blogStyleHandler), shared.NewRoute("GET", "/rss", rssBlogHandler), - shared.NewRoute("GET", "/live/rss", rssBlogHandler), shared.NewRoute("GET", "/rss.xml", rssBlogHandler), shared.NewRoute("GET", "/atom.xml", rssBlogHandler), shared.NewRoute("GET", "/feed.xml", rssBlogHandler), @@ -881,9 +874,10 @@ func createSubdomainRoutes(staticRoutes []shared.Route) []shared.Route { routes = append( routes, shared.NewRoute("GET", "/raw/(.+)", postRawHandler), - shared.NewRoute("GET", "/([^/]+)/(.+)", imgs.ImgRequest), - shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgs.ImgRequest), - shared.NewRoute("GET", "/i", imgs.ImgsListHandler), + shared.NewRoute("GET", "/(.+).md", postRawHandler), + shared.NewRoute("GET", "/([^/]+)/(.+)", imgRequest), + shared.NewRoute("GET", "/(.+.(?:jpg|jpeg|png|gif|webp|svg))$", imgRequest), + shared.NewRoute("GET", "/(.+).html", postHandler), shared.NewRoute("GET", "/(.+)", postHandler), )