feat: add support for other asset stores than S3

.env.sample

@@ -1,5 +1,4 @@
 DATABASE_URL=postgresql://user:password@localhost:5432/dbname
-REDIS_URL=redis://localhost/0
 HOME_URL=https://example.com/  # optional; if present, the home page will redirect to this URL
 SECRET_KEY=secret_key  # generate a secret key with `openssl rand -base64 32`
 LOG_LEVEL=info
@@ -12,10 +11,12 @@ LISTEN_PORT=3000
 # is implemented.
 ALLOW_PRIVATE_ADDRESS=false
 REMOTE_ACTOR_FETCH_POSTS=10
+DRIVE_DISK=
+ASSET_URL_BASE=
+FS_ASSET_PATH=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 S3_REGION=
 S3_ENDPOINT_URL=
 S3_BUCKET=
-S3_URL_BASE=
 S3_FORCE_PATH_STYLE=false

compose-fs.yaml

@@ -0,0 +1,32 @@
+services:
+  hollo:
+    image: ghcr.io/dahlia/hollo:canary
+    ports:
+      - "3000:3000"
+    environment:
+      DATABASE_URL: "postgres://user:password@postgres:5432/database"
+      SECRET_KEY: "${SECRET_KEY}"
+      LOG_LEVEL: "${LOG_LEVEL}"
+      BEHIND_PROXY: "${BEHIND_PROXY}"
+      DRIVE_DISK: fs
+      ASSET_URL_BASE: http://localhost:3000/assets/
+      FS_ASSET_PATH: /var/lib/hollo
+    depends_on:
+      - postgres
+    volumes:
+      - assets_data:/var/lib/hollo
+    restart: unless-stopped
+
+  postgres:
+    image: postgres:17
+    environment:
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: database
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    restart: unless-stopped
+
+volumes:
+  postgres_data:
+  assets_data:

compose.yaml

@@ -8,9 +8,10 @@ services:
       SECRET_KEY: "${SECRET_KEY}"
       LOG_LEVEL: "${LOG_LEVEL}"
       BEHIND_PROXY: "${BEHIND_PROXY}"
+      DRIVE_DISK: s3
+      ASSET_URL_BASE: http://localhost:9000/hollo/
       S3_REGION: us-east-1
       S3_BUCKET: hollo
-      S3_URL_BASE: http://localhost:9000/hollo/
       S3_ENDPOINT_URL: http://minio:9000
       S3_FORCE_PATH_STYLE: "true"
       AWS_ACCESS_KEY_ID: minioadmin

docs/src/content/docs/install/env.mdx

@@ -78,35 +78,52 @@ Turned off by default.
   a trusted environment and never in production.
 </Aside>
 
-### `S3_REGION` <Badge text="Optional" />
+### `DRIVE_DISK`
+
+The disk driver used by Hollo to store blobs such as avatars, custom emojis,
+and other media.
+
+Valid values are `fs` (local filesystem) and `s3` (S3-compatible object storage).
+
+See https://flydrive.dev/docs/drive_manager for details about the drivers.
+
+### `ASSET_URL_BASE`
+
+The public URL base of the asset storage, e.g.
+`https://hollo.example.com/assets` (local filesystem) or
+`https://hollo.s3.us-east-1.amazonaws.com` (Amazon S3).
+
+### `FS_ASSET_PATH`
+
+The path in the local filesystem where blob assets are stored e.g. `/var/lib/hollo`.
+The directory must be writable for Hollo.
+
+### S3-compatible object storage settings
+
+#### `S3_REGION` <Badge text="Optional" />
 
 The region of the S3-compatible object storage, e.g., `us-east-1`.  On some
 non-S3 services, this can be omitted.  `auto` by default.
 
-### `S3_BUCKET`
+#### `S3_BUCKET`
 
 The bucket name of the S3-compatible object storage, e.g., `hollo`.
 
-### `S3_URL_BASE`
-
-The public URL base of the S3-compatible object storage, e.g.,
-`https://hollo.s3.us-east-1.amazonaws.com`.
-
-### `S3_ENDPOINT_URL`
+#### `S3_ENDPOINT_URL`
 
 The endpoint URL for S3-compatible object storage, e.g.,
 `https://s3.us-east-1.amazonaws.com`.
 
-### `S3_FORCE_PATH_STYLE`
+#### `S3_FORCE_PATH_STYLE`
 
 Whether to force path-style URLs for S3-compatible object storage.  `true` to
 turn on, `false` to turn off.  Useful for non-AWS S3-compatible services.
 Turned off by default.
 
-### `AWS_ACCESS_KEY_ID`
+#### `AWS_ACCESS_KEY_ID`
 
 The access key for S3-compatible object storage.
 
-### `AWS_SECRET_ACCESS_KEY`
+#### `AWS_SECRET_ACCESS_KEY`
 
 The secret key for S3-compatible object storage

package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.577.0",
     "@aws-sdk/credential-providers": "^3.577.0",
+    "@aws-sdk/s3-request-presigner": "^3.577.0",
     "@fedify/fedify": "^1.2.2",
     "@fedify/markdown-it-hashtag": "0.2.0",
     "@fedify/markdown-it-mention": "^0.1.1",
@@ -25,10 +26,12 @@
     "drizzle-orm": "^0.30.10",
     "es-toolkit": "^1.25.2",
     "fluent-ffmpeg": "^2.1.3",
+    "flydrive": "^1.1.0",
     "hono": "^4.3.4",
     "iso-639-1": "^3.1.2",
     "markdown-it": "^14.1.0",
     "markdown-it-replace-link": "^1.2.1",
+    "mime": "^4.0.4",
     "open-graph-scraper": "^6.5.1",
     "otpauth": "^9.3.4",
     "postgres": "^3.4.5",

src/api/v1/accounts.ts

@@ -1,4 +1,3 @@
-import { PutObjectCommand } from "@aws-sdk/client-s3";
 import { Block, Undo, isActor, lookupObject } from "@fedify/fedify";
 import * as vocab from "@fedify/fedify/vocab";
 import { zValidator } from "@hono/zod-validator";
@@ -19,6 +18,7 @@ import {
   sql,
 } from "drizzle-orm";
 import { Hono } from "hono";
+import mime from "mime";
 import { z } from "zod";
 import { db } from "../../db";
 import {
@@ -38,7 +38,7 @@ import {
   unfollowAccount,
 } from "../../federation/account";
 import { type Variables, scopeRequired, tokenRequired } from "../../oauth";
-import { S3_BUCKET, S3_URL_BASE, s3 } from "../../s3";
+import { assetUrlBase, disk } from "../../s3";
 import {
   type Account,
   type AccountOwner,
@@ -117,31 +117,25 @@ app.patch(
     const form = c.req.valid("form");
     let avatarUrl = undefined;
     if (form.avatar instanceof File) {
-      await s3.send(
-        new PutObjectCommand({
-          Bucket: S3_BUCKET,
-          Key: `avatars/${account.id}`,
-          Body: new Uint8Array(await form.avatar.arrayBuffer()),
-          ContentType: form.avatar.type,
-          ACL: "public-read",
-        }),
-      );
-      avatarUrl = new URL(`avatars/${account.id}?${Date.now()}`, S3_URL_BASE)
-        .href;
+      const content = await form.avatar.arrayBuffer();
+      const path = `avatars/${account.id}.${mime.getExtension(form.avatar.type)}`;
+      await disk.put(path, new Uint8Array(content), {
+        contentType: form.avatar.type,
+        contentLength: content.byteLength,
+        visibility: "public",
+      });
+      avatarUrl = new URL(`${path}?${Date.now()}`, assetUrlBase).href;
     }
     let coverUrl = undefined;
     if (form.header instanceof File) {
-      await s3.send(
-        new PutObjectCommand({
-          Bucket: S3_BUCKET,
-          Key: `covers/${account.id}`,
-          Body: new Uint8Array(await form.header.arrayBuffer()),
-          ContentType: form.header.type,
-          ACL: "public-read",
-        }),
-      );
-      coverUrl = new URL(`covers/${account.id}?${Date.now()}`, S3_URL_BASE)
-        .href;
+      const content = await form.header.arrayBuffer();
+      const path = `covers/${account.id}.${mime.getExtension(form.header.type)}`;
+      await disk.put(path, new Uint8Array(content), {
+        contentType: form.header.type,
+        contentLength: content.byteLength,
+        visibility: "public",
+      });
+      coverUrl = new URL(`${path}?${Date.now()}`, assetUrlBase).href;
     }
     const fedCtx = federation.createContext(c.req.raw, undefined);
     const fmtOpts = {

src/api/v1/media.ts

@@ -1,13 +1,13 @@
-import { PutObjectCommand } from "@aws-sdk/client-s3";
 import { eq } from "drizzle-orm";
 import { type Context, Hono } from "hono";
+import mime from "mime";
 import sharp from "sharp";
 import { uuidv7 } from "uuidv7-js";
 import { db } from "../../db";
 import { serializeMedium } from "../../entities/medium";
 import { makeVideoScreenshot, uploadThumbnail } from "../../media";
 import { type Variables, scopeRequired, tokenRequired } from "../../oauth";
-import { S3_BUCKET, S3_URL_BASE, s3 } from "../../s3";
+import { assetUrlBase, disk } from "../../s3";
 import { media } from "../../schema";
 
 const app = new Hono<{ Variables: Variables }>();
@@ -31,16 +31,14 @@ export async function postMedia(c: Context<{ Variables: Variables }>) {
   }
   const image = sharp(imageBytes);
   const fileMetadata = await image.metadata();
-  await s3.send(
-    new PutObjectCommand({
-      Bucket: S3_BUCKET,
-      Key: `media/${id}/original`,
-      Body: new Uint8Array(fileBuffer),
-      ContentType: file.type,
-      ACL: "public-read",
-    }),
-  );
-  const url = new URL(`media/${id}/original`, S3_URL_BASE).href;
+  const content = new Uint8Array(fileBuffer);
+  const path = `media/${id}/original.${mime.getExtension(file.type)}`;
+  await disk.put(path, content, {
+    contentType: file.type,
+    contentLength: content.byteLength,
+    visibility: "public",
+  });
+  const url = new URL(path, assetUrlBase).href;
   const result = await db
     .insert(media)
     .values({

src/index.tsx

@@ -2,17 +2,28 @@ import "./logging";
 import { join } from "node:path";
 import { federation } from "@fedify/fedify/x/hono";
 import { Hono } from "hono";
+import { serveStatic } from "hono/bun";
 import { behindProxy } from "x-forwarded-fetch";
 import api from "./api";
 import fedi from "./federation";
 import image from "./image";
 import oauth, { oauthAuthorizationServer } from "./oauth";
 import pages from "./pages";
+import { DRIVE_DISK, assetPath } from "./s3";
 
 const app = new Hono();
 
-app.use(federation(fedi, (_) => undefined));
+if (DRIVE_DISK === "fs") {
+  app.use(
+    "/assets/*",
+    serveStatic({
+      root: assetPath,
+      rewriteRequestPath: (path) => path.substring("/assets".length),
+    }),
+  );
+}
 
+app.use(federation(fedi, (_) => undefined));
 app.route("/", pages);
 app.route("/oauth", oauth);
 app.get("/.well-known/oauth-authorization-server", oauthAuthorizationServer);

src/media.ts

@@ -1,10 +1,10 @@
 import { mkdtemp } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { PutObjectCommand } from "@aws-sdk/client-s3";
 import ffmpeg from "fluent-ffmpeg";
 import type { Sharp } from "sharp";
-import { S3_BUCKET, S3_URL_BASE, s3 } from "./s3";
+import { disk } from "./s3";
+import { assetUrlBase } from "./s3";
 
 const DEFAULT_THUMBNAIL_AREA = 230_400;
 
@@ -27,17 +27,14 @@ export async function uploadThumbnail(
     thumbnailArea,
   );
   const thumbnail = await original.resize(thumbnailSize).webp().toBuffer();
-  await s3.send(
-    new PutObjectCommand({
-      Bucket: S3_BUCKET,
-      Key: `media/${id}/thumbnail`,
-      Body: thumbnail.subarray(),
-      ContentType: "image/webp",
-      ACL: "public-read",
-    }),
-  );
+  const content = new Uint8Array(thumbnail);
+  await disk.put(`media/${id}/thumbnail.webp`, content, {
+    contentType: "image/webp",
+    contentLength: content.byteLength,
+    visibility: "public",
+  });
   return {
-    thumbnailUrl: new URL(`media/${id}/thumbnail`, S3_URL_BASE).href,
+    thumbnailUrl: new URL(`media/${id}/thumbnail.webp`, assetUrlBase).href,
     thumbnailType: "image/webp",
     thumbnailWidth: thumbnailSize.width,
     thumbnailHeight: thumbnailSize.height,

src/pages/emojis.tsx

@@ -1,10 +1,10 @@
-import { PutObjectCommand } from "@aws-sdk/client-s3";
 import { desc, inArray, isNotNull, ne } from "drizzle-orm";
 import { Hono } from "hono";
+import mime from "mime";
 import { DashboardLayout } from "../components/DashboardLayout";
 import db from "../db";
 import { loginRequired } from "../login";
-import { S3_BUCKET, S3_URL_BASE, s3 } from "../s3";
+import { assetUrlBase, disk } from "../s3";
 import { accounts, customEmojis, posts, reactions } from "../schema";
 
 const emojis = new Hono();
@@ -166,16 +166,14 @@ emojis.post("/", async (c) => {
   if (image == null || !(image instanceof File)) {
     return c.text("No image provided", 400);
   }
-  await s3.send(
-    new PutObjectCommand({
-      Bucket: S3_BUCKET,
-      Key: `emojis/${shortcode}`,
-      Body: new Uint8Array(await image.arrayBuffer()),
-      ContentType: image.type,
-      ACL: "public-read",
-    }),
-  );
-  const url = new URL(`emojis/${shortcode}`, S3_URL_BASE).href;
+  const content = new Uint8Array(await image.arrayBuffer());
+  const path = `emojis/${shortcode}.${mime.getExtension(image.type)}`;
+  await disk.put(path, content, {
+    contentType: image.type,
+    contentLength: content.byteLength,
+    visibility: "public",
+  });
+  const url = new URL(path, assetUrlBase).href;
   await db.insert(customEmojis).values({
     category,
     shortcode,