diff --git a/.env.example b/.env.example index 2c1acf4..1ae1b00 100644 --- a/.env.example +++ b/.env.example @@ -81,41 +81,41 @@ OPENBUCKET_REGION=us-east-1 # WEBHOOK_POLL_MS=15000 # --- async replication to an external S3 target (optional, STORY-0900) --- -# Set OB_REPLICATION_ENABLED=true to asynchronously mirror every object PUT/DELETE +# Set OPENBUCKET_REPLICATION_ENABLED=true to asynchronously mirror every object PUT/DELETE # to an external S3-compatible bucket (AWS S3 / Cloudflare R2 / Backblaze B2 / MinIO) # via a durable transactional outbox + background drain worker (per-key ordering, # last-writer-wins coalescing, exponential backoff, dead-letter cap). When enabled, -# OB_REPLICATION_BUCKET and both credentials are REQUIRED (a partial config refuses +# OPENBUCKET_REPLICATION_BUCKET and both credentials are REQUIRED (a partial config refuses # to boot). The worker sends object PLAINTEXT, so an http:// endpoint warns at boot. -# OB_REPLICATION_ENABLED=false -# OB_REPLICATION_ENDPOINT=https://.r2.cloudflarestorage.com # omit for real AWS S3 -# OB_REPLICATION_REGION=us-east-1 -# OB_REPLICATION_BUCKET=my-remote-mirror -# OB_REPLICATION_ACCESS_KEY_ID= -# OB_REPLICATION_SECRET_ACCESS_KEY= -# OB_REPLICATION_FORCE_PATH_STYLE=true # true for MinIO/S3-compat; false for AWS +# OPENBUCKET_REPLICATION_ENABLED=false +# OPENBUCKET_REPLICATION_ENDPOINT=https://.r2.cloudflarestorage.com # omit for real AWS S3 +# OPENBUCKET_REPLICATION_REGION=us-east-1 +# OPENBUCKET_REPLICATION_BUCKET=my-remote-mirror +# OPENBUCKET_REPLICATION_ACCESS_KEY_ID= +# OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY= +# OPENBUCKET_REPLICATION_FORCE_PATH_STYLE=true # true for MinIO/S3-compat; false for AWS # Drain tuning (defaults shown). -# OB_REPLICATION_MAX_ATTEMPTS=12 -# OB_REPLICATION_DRAIN_INTERVAL_MS=5000 -# OB_REPLICATION_BATCH_KEYS=50 -# OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES=67108864 +# OPENBUCKET_REPLICATION_MAX_ATTEMPTS=12 +# OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS=5000 +# OPENBUCKET_REPLICATION_BATCH_KEYS=50 +# OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES=67108864 # --- scheduled backups & retention (optional, STORY-1203) --- -# Set OB_SCHEDULED_BACKUP_ENABLED=true to write a .zip snapshot (identical to the +# Set OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true to write a .zip snapshot (identical to the # admin download) on a schedule, with retention. Exactly ONE of INTERVAL_MINUTES / # CRON must be set (both/neither refuses to boot; a malformed cron fails fast). # Snapshots hold decrypted plaintext object bytes — the backup dir (default # /backups, files 0o600 under a 0o700 dir) inherits the data volume's # trust boundary. Retention is union (keep-last OR max-age); disk growth is bounded. -# OB_SCHEDULED_BACKUP_ENABLED=false -# OB_SCHEDULED_BACKUP_SCOPE=instance # instance = one snapshot; buckets = one per bucket -# OB_SCHEDULED_BACKUP_INTERVAL_MINUTES=1440 # every 24h (mutually exclusive with CRON) -# OB_SCHEDULED_BACKUP_CRON=0 3 * * * # 5-field cron (mutually exclusive with INTERVAL) -# OB_SCHEDULED_BACKUP_DIR=/data/backups # default /backups -# OB_SCHEDULED_BACKUP_KEEP_LAST=7 # keep the newest N (a hard floor) -# OB_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # also keep anything younger than this -# OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS=60000 # wake tick — how often "is a snapshot due?" -# OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=false # also push the .zip to the replication target +# OPENBUCKET_SCHEDULED_BACKUP_ENABLED=false +# OPENBUCKET_SCHEDULED_BACKUP_SCOPE=instance # instance = one snapshot; buckets = one per bucket +# OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES=1440 # every 24h (mutually exclusive with CRON) +# OPENBUCKET_SCHEDULED_BACKUP_CRON=0 3 * * * # 5-field cron (mutually exclusive with INTERVAL) +# OPENBUCKET_SCHEDULED_BACKUP_DIR=/data/backups # default /backups +# OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST=7 # keep the newest N (a hard floor) +# OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # also keep anything younger than this +# OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS=60000 # wake tick — how often "is a snapshot due?" +# OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=false # also push the .zip to the replication target # --- shutdown (optional) --- # SHUTDOWN_DRAIN_MS=30000 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1810345..e88b14e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,23 @@ versions may include breaking changes. ## [Unreleased] +### Changed + +- **BREAKING (pre-1.0): standalone env-var prefix unified onto `OPENBUCKET_`.** + The replication, scheduled-backup, and integrity-scrub feature blocks used an + `OB_` prefix while every other feature block (tiering, endpoint, region, SSE) + used `OPENBUCKET_`. They are now consistent. Rename in your `.env` / compose / + deploy config: + - `OB_REPLICATION_*` → `OPENBUCKET_REPLICATION_*` + - `OB_SCHEDULED_BACKUP_*` → `OPENBUCKET_SCHEDULED_BACKUP_*` + - `OB_INTEGRITY_*` → `OPENBUCKET_INTEGRITY_*` + + This is a hard rename with **no backward-compatible aliases** — the old names + are no longer read. The bare core vars (`DATA_DIR`, `JWT_SECRET`, + `ROOT_ACCESS_KEY_ID`, `PORT`, …) and the already-bare feature knobs + (`WEBHOOK_*`, `METRICS_*`, `RESTORE_*`, …) are unchanged. The programmatic + `OpenBucketModule.forRoot` API is unaffected. + ## [0.1.0-alpha.18] — 2026-07-08 ### Added @@ -168,7 +185,7 @@ Feature release — durability & cloud replication (EPIC-10). **transactional outbox** and mirrored by a background worker with per-key ordering, last-writer-wins coalescing, exponential-backoff retry, and a dead-letter cap — resuming on boot and surviving remote outages. Configure via - `OB_REPLICATION_*` env vars or the `replication` module option. + `OPENBUCKET_REPLICATION_*` env vars or the `replication` module option. - **Cold-object tiering** — objects not accessed within a policy window are offloaded to the replication target to free local disk; a `GET` transparently rehydrates them (read-through). Lifecycle `` rules now drive tiering. diff --git a/README.md b/README.md index f936afb..dc3cf48 100644 --- a/README.md +++ b/README.md @@ -395,27 +395,27 @@ full admin. See the ### Async replication -Set `OB_REPLICATION_ENABLED=true` to asynchronously mirror every object +Set `OPENBUCKET_REPLICATION_ENABLED=true` to asynchronously mirror every object mutation to an external S3-compatible bucket (see the embedded [library README](./libs/nestjs/README.md#async-replication-to-an-external-s3-target) -for how it works). When enabled, `OB_REPLICATION_BUCKET` and both credentials are +for how it works). When enabled, `OPENBUCKET_REPLICATION_BUCKET` and both credentials are required together (a partial config refuses to boot). | Variable | Required\* | Default | Notes | | --------------------------------------- | ---------- | ------------- | --------------------------------------------------------- | -| `OB_REPLICATION_ENABLED` | | `false` | Master switch. Off ⇒ zero cost, outbox stays empty. | -| `OB_REPLICATION_ENDPOINT` | | — | S3-compatible endpoint (R2/B2/MinIO). Omit for real AWS S3. `http://` warns (plaintext). | -| `OB_REPLICATION_REGION` | | `us-east-1` | Target region. | -| `OB_REPLICATION_BUCKET` | ✅ | — | Remote target bucket (must already exist). | -| `OB_REPLICATION_ACCESS_KEY_ID` | ✅ | — | Target credential. | -| `OB_REPLICATION_SECRET_ACCESS_KEY` | ✅ | — | Target credential (never logged; redacted). | -| `OB_REPLICATION_FORCE_PATH_STYLE` | | `true` | `true` for MinIO/S3-compat; `false` for AWS. | -| `OB_REPLICATION_MAX_ATTEMPTS` | | `12` | Dead-letter cap before an intent → `failed`. | -| `OB_REPLICATION_DRAIN_INTERVAL_MS` | | `5000` | Background drain tick interval (≥ 1000). | -| `OB_REPLICATION_BATCH_KEYS` | | `50` | Distinct keys drained per tick. | -| `OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` | | `67108864` | Stream via multipart above this (64 MiB). | - -\* Required only when `OB_REPLICATION_ENABLED=true`. +| `OPENBUCKET_REPLICATION_ENABLED` | | `false` | Master switch. Off ⇒ zero cost, outbox stays empty. | +| `OPENBUCKET_REPLICATION_ENDPOINT` | | — | S3-compatible endpoint (R2/B2/MinIO). Omit for real AWS S3. `http://` warns (plaintext). | +| `OPENBUCKET_REPLICATION_REGION` | | `us-east-1` | Target region. | +| `OPENBUCKET_REPLICATION_BUCKET` | ✅ | — | Remote target bucket (must already exist). | +| `OPENBUCKET_REPLICATION_ACCESS_KEY_ID` | ✅ | — | Target credential. | +| `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` | ✅ | — | Target credential (never logged; redacted). | +| `OPENBUCKET_REPLICATION_FORCE_PATH_STYLE` | | `true` | `true` for MinIO/S3-compat; `false` for AWS. | +| `OPENBUCKET_REPLICATION_MAX_ATTEMPTS` | | `12` | Dead-letter cap before an intent → `failed`. | +| `OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS` | | `5000` | Background drain tick interval (≥ 1000). | +| `OPENBUCKET_REPLICATION_BATCH_KEYS` | | `50` | Distinct keys drained per tick. | +| `OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` | | `67108864` | Stream via multipart above this (64 MiB). | + +\* Required only when `OPENBUCKET_REPLICATION_ENABLED=true`. The admin console's **Replication** page (and the `/api/admin/replication` API) shows replication health — pending/failed depth and lag — and offers a diff --git a/apps/docs/docs/concepts/durability.md b/apps/docs/docs/concepts/durability.md index 6ba8653..62ab9e4 100644 --- a/apps/docs/docs/concepts/durability.md +++ b/apps/docs/docs/concepts/durability.md @@ -74,8 +74,8 @@ so it never starves request traffic. ```bash # Enable the scrubber (standalone): -OB_INTEGRITY_SCRUB_ENABLED=true -OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK=1073741824 # 1 GiB/tick (default) +OPENBUCKET_INTEGRITY_SCRUB_ENABLED=true +OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK=1073741824 # 1 GiB/tick (default) ``` When a blob is `corrupt` **and** a replication target is configured, the scrubber @@ -97,11 +97,11 @@ resumes on boot after a crash or a remote outage — local reads and writes keep working while the remote is unreachable, and the backlog drains on recovery. ```bash -OB_REPLICATION_ENABLED=true -OB_REPLICATION_BUCKET=my-remote-mirror # must already exist -OB_REPLICATION_ACCESS_KEY_ID=… -OB_REPLICATION_SECRET_ACCESS_KEY=… -# Omit OB_REPLICATION_ENDPOINT for real AWS S3; set it for R2/B2/MinIO. +OPENBUCKET_REPLICATION_ENABLED=true +OPENBUCKET_REPLICATION_BUCKET=my-remote-mirror # must already exist +OPENBUCKET_REPLICATION_ACCESS_KEY_ID=… +OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY=… +# Omit OPENBUCKET_REPLICATION_ENDPOINT for real AWS S3; set it for R2/B2/MinIO. ``` :::note[Plaintext over the wire] @@ -132,11 +132,11 @@ Beyond the on-demand backup/restore endpoints, OpenBucket can write **`.zip` snapshots on a schedule** and prune them by a retention policy: ```bash -OB_SCHEDULED_BACKUP_ENABLED=true -OB_SCHEDULED_BACKUP_SCOPE=instance # or 'buckets' (one .zip per bucket) -OB_SCHEDULED_BACKUP_CRON='0 3 * * *' # OR OB_SCHEDULED_BACKUP_INTERVAL_MINUTES -OB_SCHEDULED_BACKUP_KEEP_LAST=7 # retention floor -OB_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # union with keep-last +OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true +OPENBUCKET_SCHEDULED_BACKUP_SCOPE=instance # or 'buckets' (one .zip per bucket) +OPENBUCKET_SCHEDULED_BACKUP_CRON='0 3 * * *' # OR OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES +OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST=7 # retention floor +OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # union with keep-last ``` Each snapshot is written **atomically** (stream to `.part`, fsync, rename) diff --git a/apps/docs/docs/guides/backup-and-restore.md b/apps/docs/docs/guides/backup-and-restore.md index c04d032..2aae6d3 100644 --- a/apps/docs/docs/guides/backup-and-restore.md +++ b/apps/docs/docs/guides/backup-and-restore.md @@ -78,14 +78,14 @@ snapshot to your replication target. ### Standalone (env) ```bash -OB_SCHEDULED_BACKUP_ENABLED=true -OB_SCHEDULED_BACKUP_SCOPE=instance # or `buckets` (one .zip per bucket) -OB_SCHEDULED_BACKUP_CRON="0 3 * * *" # 03:00 daily — XOR with INTERVAL_MINUTES -OB_SCHEDULED_BACKUP_KEEP_LAST=7 # keep the newest N (default 7) -OB_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # also keep anything younger (default 30) -OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=false -# OB_SCHEDULED_BACKUP_DIR=/var/lib/openbucket/backups # default /backups -# OB_SCHEDULED_BACKUP_INTERVAL_MINUTES=1440 # instead of a cron +OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true +OPENBUCKET_SCHEDULED_BACKUP_SCOPE=instance # or `buckets` (one .zip per bucket) +OPENBUCKET_SCHEDULED_BACKUP_CRON="0 3 * * *" # 03:00 daily — XOR with INTERVAL_MINUTES +OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST=7 # keep the newest N (default 7) +OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS=30 # also keep anything younger (default 30) +OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=false +# OPENBUCKET_SCHEDULED_BACKUP_DIR=/var/lib/openbucket/backups # default /backups +# OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES=1440 # instead of a cron ``` ### Embedded (`forRoot`) @@ -136,7 +136,7 @@ it can never launch a second overlapping snapshot — a concurrent request retur ## Push snapshots off-box -Set `pushToReplication` (env `OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true`) to +Set `pushToReplication` (env `OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true`) to copy each finished snapshot to your configured [replication target](./replication-and-tiering.md) under a reserved prefix — so a lost data volume doesn't take your backups with it. @@ -144,7 +144,7 @@ volume doesn't take your backups with it. :::note[Enable replication first] If you set the push flag but replication is off, the runner logs a boot-time warning and writes snapshots locally only — the push is a no-op until you -configure `OB_REPLICATION_*`. +configure `OPENBUCKET_REPLICATION_*`. ::: :::warning[Snapshots are plaintext] diff --git a/apps/docs/docs/guides/replication-and-tiering.md b/apps/docs/docs/guides/replication-and-tiering.md index f0d0367..4004e12 100644 --- a/apps/docs/docs/guides/replication-and-tiering.md +++ b/apps/docs/docs/guides/replication-and-tiering.md @@ -23,16 +23,16 @@ target is a durable copy. ### Standalone (env) ```bash -OB_REPLICATION_ENABLED=true -OB_REPLICATION_BUCKET=openbucket-mirror -OB_REPLICATION_ACCESS_KEY_ID=... -OB_REPLICATION_SECRET_ACCESS_KEY=... -OB_REPLICATION_ENDPOINT=https://.r2.cloudflarestorage.com # omit for AWS S3 -OB_REPLICATION_REGION=us-east-1 -OB_REPLICATION_FORCE_PATH_STYLE=true # true for MinIO/R2/B2; false for AWS S3 +OPENBUCKET_REPLICATION_ENABLED=true +OPENBUCKET_REPLICATION_BUCKET=openbucket-mirror +OPENBUCKET_REPLICATION_ACCESS_KEY_ID=... +OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY=... +OPENBUCKET_REPLICATION_ENDPOINT=https://.r2.cloudflarestorage.com # omit for AWS S3 +OPENBUCKET_REPLICATION_REGION=us-east-1 +OPENBUCKET_REPLICATION_FORCE_PATH_STYLE=true # true for MinIO/R2/B2; false for AWS S3 ``` -`OB_REPLICATION_BUCKET` and both credentials are required when replication is on — +`OPENBUCKET_REPLICATION_BUCKET` and both credentials are required when replication is on — a partial config fails at boot. The target bucket must already exist. ### Embedded (`forRoot`) @@ -44,8 +44,8 @@ OpenBucketModule.forRoot({ replication: { bucket: 'openbucket-mirror', credentials: { - accessKeyId: process.env.OB_REPLICATION_ACCESS_KEY_ID!, - secretAccessKey: process.env.OB_REPLICATION_SECRET_ACCESS_KEY!, + accessKeyId: process.env.OPENBUCKET_REPLICATION_ACCESS_KEY_ID!, + secretAccessKey: process.env.OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY!, }, endpoint: 'https://.r2.cloudflarestorage.com', // omit for AWS S3 forcePathStyle: true, @@ -66,14 +66,14 @@ background drain worker then mirrors those intents to the remote: - **Last-writer-wins coalescing** — a burst of writes to one key collapses to the latest state. - **Retry with exponential backoff** — a transient remote failure is retried up - to a **dead-letter cap** (`OB_REPLICATION_MAX_ATTEMPTS`, default 12). + to a **dead-letter cap** (`OPENBUCKET_REPLICATION_MAX_ATTEMPTS`, default 12). - **Resume on boot** — the worker picks up pending intents after a restart and survives remote outages: nothing is lost, it just backs up in the outbox and drains when the remote returns. -Tuning knobs: `OB_REPLICATION_DRAIN_INTERVAL_MS` (default 5000), -`OB_REPLICATION_BATCH_KEYS` (distinct keys per tick, default 50), and -`OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` (switch to multipart streaming above +Tuning knobs: `OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS` (default 5000), +`OPENBUCKET_REPLICATION_BATCH_KEYS` (distinct keys per tick, default 50), and +`OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` (switch to multipart streaming above this size, default 64 MiB). :::warning[The wire carries plaintext] diff --git a/apps/docs/docs/guides/securing-openbucket.md b/apps/docs/docs/guides/securing-openbucket.md index a6fcd9f..7878bff 100644 --- a/apps/docs/docs/guides/securing-openbucket.md +++ b/apps/docs/docs/guides/securing-openbucket.md @@ -144,10 +144,10 @@ catch bit-rot or tampering at rest. It's **off by default** and strictly rate limited so it never starves request traffic: ```bash -OB_INTEGRITY_SCRUB_ENABLED=true -OB_INTEGRITY_SCRUB_INTERVAL_MS=60000 # tick cadence -OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK=1000 # per-tick object budget -OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK=... # per-tick byte budget +OPENBUCKET_INTEGRITY_SCRUB_ENABLED=true +OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS=60000 # tick cadence +OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK=1000 # per-tick object budget +OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK=... # per-tick byte budget ``` Each object gets a verdict (`unchecked` / `ok` / `corrupt`). When a diff --git a/apps/docs/docs/reference/configuration.md b/apps/docs/docs/reference/configuration.md index a45ad1e..c8bbe60 100644 --- a/apps/docs/docs/reference/configuration.md +++ b/apps/docs/docs/reference/configuration.md @@ -134,24 +134,24 @@ The URL must be `https` unless the host is loopback; the secret is then required ### Async replication Mirror every object mutation to an external S3-compatible target. When -`OB_REPLICATION_ENABLED=true`, the bucket and both credentials are **required +`OPENBUCKET_REPLICATION_ENABLED=true`, the bucket and both credentials are **required together** — a partial config refuses to boot. | Variable | Required* | Default | Notes | | --- | :-: | --- | --- | -| `OB_REPLICATION_ENABLED` | | `false` | Master switch. Off ⇒ zero cost, outbox stays empty. | -| `OB_REPLICATION_ENDPOINT` | | — | S3-compatible endpoint (R2/B2/MinIO). Omit for real AWS S3. `http://` warns at boot (plaintext). | -| `OB_REPLICATION_REGION` | | `us-east-1` | Target region. | -| `OB_REPLICATION_BUCKET` | ✅ | — | Remote target bucket (must already exist). | -| `OB_REPLICATION_ACCESS_KEY_ID` | ✅ | — | Target credential. | -| `OB_REPLICATION_SECRET_ACCESS_KEY` | ✅ | — | Target credential (never logged). | -| `OB_REPLICATION_FORCE_PATH_STYLE` | | `true` | `true` for MinIO/S3-compat; `false` for AWS. | -| `OB_REPLICATION_MAX_ATTEMPTS` | | `12` | Dead-letter cap (1–50). | -| `OB_REPLICATION_DRAIN_INTERVAL_MS` | | `5000` | Drain tick interval (1000–300000). | -| `OB_REPLICATION_BATCH_KEYS` | | `50` | Distinct keys drained per tick (1–1000). | -| `OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` | | `67108864` | Stream via multipart above this (64 MiB). | - -\* Required only when `OB_REPLICATION_ENABLED=true`. +| `OPENBUCKET_REPLICATION_ENABLED` | | `false` | Master switch. Off ⇒ zero cost, outbox stays empty. | +| `OPENBUCKET_REPLICATION_ENDPOINT` | | — | S3-compatible endpoint (R2/B2/MinIO). Omit for real AWS S3. `http://` warns at boot (plaintext). | +| `OPENBUCKET_REPLICATION_REGION` | | `us-east-1` | Target region. | +| `OPENBUCKET_REPLICATION_BUCKET` | ✅ | — | Remote target bucket (must already exist). | +| `OPENBUCKET_REPLICATION_ACCESS_KEY_ID` | ✅ | — | Target credential. | +| `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` | ✅ | — | Target credential (never logged). | +| `OPENBUCKET_REPLICATION_FORCE_PATH_STYLE` | | `true` | `true` for MinIO/S3-compat; `false` for AWS. | +| `OPENBUCKET_REPLICATION_MAX_ATTEMPTS` | | `12` | Dead-letter cap (1–50). | +| `OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS` | | `5000` | Drain tick interval (1000–300000). | +| `OPENBUCKET_REPLICATION_BATCH_KEYS` | | `50` | Distinct keys drained per tick (1–1000). | +| `OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES` | | `67108864` | Stream via multipart above this (64 MiB). | + +\* Required only when `OPENBUCKET_REPLICATION_ENABLED=true`. ### Cold-object tiering @@ -169,20 +169,20 @@ read. A no-op unless a replication target is also configured. ### Scheduled backups Write a `.zip` snapshot on a schedule with union retention. When -`OB_SCHEDULED_BACKUP_ENABLED=true`, **exactly one** of `INTERVAL_MINUTES` / +`OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true`, **exactly one** of `INTERVAL_MINUTES` / `CRON` must be set. | Variable | Default | Notes | | --- | --- | --- | -| `OB_SCHEDULED_BACKUP_ENABLED` | `false` | Master switch. | -| `OB_SCHEDULED_BACKUP_SCOPE` | `instance` | `instance` = one snapshot; `buckets` = one per bucket. | -| `OB_SCHEDULED_BACKUP_INTERVAL_MINUTES` | — | Fixed interval (5–43200). Mutually exclusive with `CRON`. | -| `OB_SCHEDULED_BACKUP_CRON` | — | 5-field cron. Mutually exclusive with `INTERVAL_MINUTES`. | -| `OB_SCHEDULED_BACKUP_DIR` | `/backups` | Absolute snapshot directory. | -| `OB_SCHEDULED_BACKUP_KEEP_LAST` | `7` | Keep the newest N snapshots (hard floor, 1–1000). | -| `OB_SCHEDULED_BACKUP_MAX_AGE_DAYS` | `30` | Also keep anything younger than this (union, 1–3650). | -| `OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS` | `60000` | "Is a snapshot due?" wake tick (10000–3600000). | -| `OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION` | `false` | Also push each `.zip` to the replication target. | +| `OPENBUCKET_SCHEDULED_BACKUP_ENABLED` | `false` | Master switch. | +| `OPENBUCKET_SCHEDULED_BACKUP_SCOPE` | `instance` | `instance` = one snapshot; `buckets` = one per bucket. | +| `OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES` | — | Fixed interval (5–43200). Mutually exclusive with `CRON`. | +| `OPENBUCKET_SCHEDULED_BACKUP_CRON` | — | 5-field cron. Mutually exclusive with `INTERVAL_MINUTES`. | +| `OPENBUCKET_SCHEDULED_BACKUP_DIR` | `/backups` | Absolute snapshot directory. | +| `OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST` | `7` | Keep the newest N snapshots (hard floor, 1–1000). | +| `OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS` | `30` | Also keep anything younger than this (union, 1–3650). | +| `OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS` | `60000` | "Is a snapshot due?" wake tick (10000–3600000). | +| `OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION` | `false` | Also push each `.zip` to the replication target. | ### Integrity scrubbing @@ -191,10 +191,10 @@ detect bit-rot. Strictly rate-limited so it never starves request traffic. | Variable | Default | Notes | | --- | --- | --- | -| `OB_INTEGRITY_SCRUB_ENABLED` | `false` | Master switch. A fresh install does zero extra I/O. | -| `OB_INTEGRITY_SCRUB_INTERVAL_MS` | `60000` | Tick interval (floor `1000`). | -| `OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` | `1000` | Hard per-tick object cap (min `1`). | -| `OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` | `1073741824` | Per-tick byte budget (1 GiB). | +| `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` | `false` | Master switch. A fresh install does zero extra I/O. | +| `OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS` | `60000` | Tick interval (floor `1000`). | +| `OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` | `1000` | Hard per-tick object cap (min `1`). | +| `OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` | `1073741824` | Per-tick byte budget (1 GiB). | ### Restore caps & shutdown @@ -214,6 +214,18 @@ detect bit-rot. Strictly rate-limited so it never starves request traffic. When you embed OpenBucket, pass these to `OpenBucketModule.forRoot(...)` instead of environment variables. Only `dataDir` and `rootCredentials` are required. +:::note[Some standalone env features are env-only] +The `forRoot` options are a deliberate subset of the standalone environment +variables — a few features are configurable **only** via the environment and have +no `forRoot` option counterpart: + +- `KEY_ENCRYPTION_SECRET` — KEK material for scoped sub-key secrets at rest. +- the cold-object tiering family (`OPENBUCKET_TIER_*`). + +When embedding, set these through `process.env` (or your app's config) even though +they are not part of the `forRoot` options object. +::: + ```ts OpenBucketModule.forRoot({ dataDir: '/var/lib/openbucket', diff --git a/apps/docs/docs/reference/nestjs-module.md b/apps/docs/docs/reference/nestjs-module.md index 53933ae..b4411d2 100644 --- a/apps/docs/docs/reference/nestjs-module.md +++ b/apps/docs/docs/reference/nestjs-module.md @@ -322,6 +322,14 @@ Notes: `forRootAsync` adds two **static** options alongside `useFactory`/`inject`: `serveUi?` (default `true`) and `admin?` (default `true` — set `false` for headless). +:::note[Env-only features] +These options are a deliberate subset of the standalone environment variables. A +few features are configurable **only** through the environment and have no +`forRoot` counterpart — notably `KEY_ENCRYPTION_SECRET` and the cold-object +tiering family (`OPENBUCKET_TIER_*`). Set those via `process.env` even when +embedding. See the [configuration reference](./configuration.md). +::: + ## How it coexists with your app - **Mounting.** Everything mounts under `mountPath`, so OpenBucket's greedy S3 diff --git a/apps/openbucket-frontend/src/app/i18n/de.translations.ts b/apps/openbucket-frontend/src/app/i18n/de.translations.ts index 6f0be12..8bfe61c 100644 --- a/apps/openbucket-frontend/src/app/i18n/de.translations.ts +++ b/apps/openbucket-frontend/src/app/i18n/de.translations.ts @@ -167,7 +167,7 @@ export default { runStarted: 'Sicherung gestartet', runJoined: 'Eine Sicherung läuft bereits', runFailed: 'Sicherung konnte nicht gestartet werden', - offHint: 'Geplante Sicherungen sind deaktiviert. Setze OB_SCHEDULED_BACKUP_*, um sie zu aktivieren.', + offHint: 'Geplante Sicherungen sind deaktiviert. Setze OPENBUCKET_SCHEDULED_BACKUP_*, um sie zu aktivieren.', }, }, breadcrumb: { diff --git a/apps/openbucket-frontend/src/app/i18n/en.translations.ts b/apps/openbucket-frontend/src/app/i18n/en.translations.ts index 8032ff5..48c0a71 100644 --- a/apps/openbucket-frontend/src/app/i18n/en.translations.ts +++ b/apps/openbucket-frontend/src/app/i18n/en.translations.ts @@ -165,7 +165,7 @@ export default { runStarted: 'Backup started', runJoined: 'A backup is already running', runFailed: 'Could not start backup', - offHint: 'Scheduled backups are disabled. Set OB_SCHEDULED_BACKUP_* to enable.', + offHint: 'Scheduled backups are disabled. Set OPENBUCKET_SCHEDULED_BACKUP_* to enable.', }, }, breadcrumb: { diff --git a/docs/pm/stories/STORY-0900-async-replication-to-external-s3-target.md b/docs/pm/stories/STORY-0900-async-replication-to-external-s3-target.md index 7f2d9c5..1b85212 100644 --- a/docs/pm/stories/STORY-0900-async-replication-to-external-s3-target.md +++ b/docs/pm/stories/STORY-0900-async-replication-to-external-s3-target.md @@ -97,17 +97,17 @@ scripts were removed at close; the cases were promoted into the unit specs): - **Large objects** — the object size is always known (from the object row), so a plain `PutObjectCommand` with `ContentLength` is used below the threshold and `@aws-sdk/lib-storage` `Upload` (streaming multipart) above it. Threshold: - `largeObjectThresholdBytes = 64 MiB` (`OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES`). + `largeObjectThresholdBytes = 64 MiB` (`OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES`). - **Crash-resume / idempotency** — no persisted `inflight` state is needed (single process + the scheduler's no-pileup guard). A crash mid-send leaves the intent `pending`; the first tick after boot re-sends it. PUT is idempotent by key, DELETE is idempotent (a remote 404 is success), so a re-send is safe. - **Retry realism** — full-jitter exponential backoff `min(1s * 2^(n-1), 5min) * rand(0.5..1.5)`; dead-letter to `status='failed'` after - `maxAttempts = 12` (`OB_REPLICATION_MAX_ATTEMPTS`). The SDK's own retry is + `maxAttempts = 12` (`OPENBUCKET_REPLICATION_MAX_ATTEMPTS`). The SDK's own retry is disabled (`maxAttempts: 1`) so the worker owns the single retry budget. - **Security** — an `http://` endpoint logs a boot-time warning (replicated - bytes are object plaintext); `OB_REPLICATION_SECRET_ACCESS_KEY` / + bytes are object plaintext); `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` / `secretAccessKey` / `authorization` are in the pino redact paths. ## References diff --git a/docs/pm/stories/STORY-1203-scheduled-backups-and-retention.md b/docs/pm/stories/STORY-1203-scheduled-backups-and-retention.md index 53537ec..022e4ed 100644 --- a/docs/pm/stories/STORY-1203-scheduled-backups-and-retention.md +++ b/docs/pm/stories/STORY-1203-scheduled-backups-and-retention.md @@ -34,14 +34,14 @@ secrets in `/metrics` or logs, and JWT-guarded admin endpoints. ## Acceptance criteria - [ ] `BackupService` exposes a sink-based snapshot method (`writeSnapshot(sink, kind, bucketNames)` returning `{ bytes, objectCount }`); `streamBucketBackup` / `streamInstanceBackup` are refactored to call it so the streamed-download and scheduled-file paths share one code path and one `BackupManifest` writer. -- [ ] With `OB_SCHEDULED_BACKUP_ENABLED=false` (default) no snapshot is written and the runner is a no-op (registered unconditionally, mirrors `ReplicationWorkerRunner.run`'s `if (!enabled) return`). -- [ ] With scheduling enabled and `OB_SCHEDULED_BACKUP_INTERVAL_MINUTES` (or `OB_SCHEDULED_BACKUP_CRON`) set, a snapshot `.zip` + sidecar `.json` metadata appears under `//` on the configured cadence, written atomically (`.part` → `fsync` → `rename`) with mode `0o600`. +- [ ] With `OPENBUCKET_SCHEDULED_BACKUP_ENABLED=false` (default) no snapshot is written and the runner is a no-op (registered unconditionally, mirrors `ReplicationWorkerRunner.run`'s `if (!enabled) return`). +- [ ] With scheduling enabled and `OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES` (or `OPENBUCKET_SCHEDULED_BACKUP_CRON`) set, a snapshot `.zip` + sidecar `.json` metadata appears under `//` on the configured cadence, written atomically (`.part` → `fsync` → `rename`) with mode `0o600`. - [ ] Retention prunes snapshots: an entry is retained iff it is among the newest `keepLast` **or** younger than `maxAgeDays`; it is deleted only when it fails both — so keep-last-N never deletes below the floor even for old entries, and max-age never deletes a fresh entry. -- [ ] When `OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true` **and** replication is enabled, each new snapshot `.zip` is uploaded via `ReplicationTargetService.putObject` under the reserved `_ob_backups/` prefix (multipart for large archives); when replication is disabled the flag is ignored with a boot-time warning. +- [ ] When `OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true` **and** replication is enabled, each new snapshot `.zip` is uploaded via `ReplicationTargetService.putObject` under the reserved `_ob_backups/` prefix (multipart for large archives); when replication is disabled the flag is ignored with a boot-time warning. - [ ] `GET /api/admin/backup/schedule` returns `{ enabled, scope, schedule, lastRunAt, nextRunAt, lastStatus, lastError, lastDurationMs, lastBytes, lastObjectCount, keepLast, maxAgeDays, snapshotCount }` — no directory paths, credentials, or object keys in the payload. - [ ] `POST /api/admin/backup/schedule/run-now` triggers exactly one snapshot + prune, guarded by an in-flight lock so it can neither overlap a scheduled tick nor be flooded into N concurrent snapshots; both endpoints are behind the global admin JWT guard. - [ ] The Angular `backup-restore.component.ts` shows last-run / next-run and a "Run now" button (signals-based, `OnPush`), refreshing status after the run resolves. -- [ ] A malformed `OB_SCHEDULED_BACKUP_CRON` fails fast at boot (env schema / `validateSecurityCriticalOptions`), never mid-tick; a snapshot is skipped (with a warning, not a crash) when free space is below `DATA_DIR_MIN_FREE_BYTES`. +- [ ] A malformed `OPENBUCKET_SCHEDULED_BACKUP_CRON` fails fast at boot (env schema / `validateSecurityCriticalOptions`), never mid-tick; a snapshot is skipped (with a warning, not a crash) when free space is below `DATA_DIR_MIN_FREE_BYTES`. - [ ] `/metrics` (STORY-1202) exposes only a last-success timestamp / last-bytes / snapshot-count gauge for backups — never a path, key, or secret — consistent with `RequestMetricsService`'s counts-only posture. ## Tasks @@ -71,4 +71,4 @@ secrets in `/metrics` or logs, and JWT-guarded admin endpoints. - `libs/nestjs/src/lib/common/config/env.schema.ts`, `app-config.service.ts`, `config-source.ts` - `apps/openbucket-frontend/src/app/backup-restore/backup-restore.component.ts` - `apps/openbucket-backend/webpack.config.js` (`externalDependencies`) — 3-place externalization for the new `cron-parser` dep (also `libs/nestjs/package.json` + `apps/openbucket-backend/package.json`) -- New dep: `cron-parser` (pure-JS cron expression → next-run computation; only required when `OB_SCHEDULED_BACKUP_CRON` is used) +- New dep: `cron-parser` (pure-JS cron expression → next-run computation; only required when `OPENBUCKET_SCHEDULED_BACKUP_CRON` is used) diff --git a/docs/pm/stories/STORY-1204-integrity-scrubbing.md b/docs/pm/stories/STORY-1204-integrity-scrubbing.md index bfe48bd..62a4183 100644 --- a/docs/pm/stories/STORY-1204-integrity-scrubbing.md +++ b/docs/pm/stories/STORY-1204-integrity-scrubbing.md @@ -27,10 +27,10 @@ budget, `setImmediate` yields) so it never competes with foreground I/O. ## Acceptance criteria - [ ] A new `IntegrityScrubRunner implements ScheduledTask` is registered in `BackgroundModule` (providers + the `SCHEDULED_TASKS` factory `inject` list) and runs on its own interval via the existing tick scheduler. -- [ ] The scrub is OFF by default: with `OB_INTEGRITY_SCRUB_ENABLED` unset, `run()` returns immediately and performs zero disk reads or DB writes (mirrors `TieringSweepRunner`'s default-off gate). +- [ ] The scrub is OFF by default: with `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` unset, `run()` returns immediately and performs zero disk reads or DB writes (mirrors `TieringSweepRunner`'s default-off gate). - [ ] For each scanned current, local, non-soft-deleted object with a non-null `contentSha256`, the runner re-hashes the on-disk blob (decrypting SSE via `createSseDecipher` when `encryption` is set) and persists `integrityStatus` + `integrityCheckedAt`. - [ ] A blob whose recomputed SHA-256 differs from the stored digest is marked `corrupt` (never served — the F1 read gate already 500s it) and, when `ReplicationTargetService.enabled`, repaired by streaming the remote copy, staging it via `BlobStore` (tmp → fsync → atomic rename), re-verifying, and only then flipping the row back to `ok`. -- [ ] The scrub is bounded per tick: at most `INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` objects and `OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` bytes are hashed, it yields to the event loop between batches, and it resumes from a persisted cursor next tick — a multi-TB store is walked incrementally without pinning the EM or event loop. +- [ ] The scrub is bounded per tick: at most `INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` objects and `OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` bytes are hashed, it yields to the event loop between batches, and it resumes from a persisted cursor next tick — a multi-TB store is walked incrementally without pinning the EM or event loop. - [ ] `GET /api/admin/integrity/status` (JwtAuthGuard, `default` throttler) returns `{ enabled, scanned, ok, corrupt, unchecked, repaired, lastRunAt, cursor }` and `GET /api/admin/integrity/corrupt` returns a bounded, paged list of corrupt objects `{ bucket, key, checkedAt, detail }` — never any remote endpoint/credential. - [ ] The admin console shows a corruption indicator (a red badge with the corrupt count) sourced from an Angular signal store; it reads `getIntegrityStatus` and shows a not-configured/clean state when `corrupt === 0`. - [ ] Per-object failures (ENOENT, remote outage, re-verify mismatch) are isolated: caught, logged with a redacted message, the cursor still advances, and the tick never throws. diff --git a/docs/pm/tasks/TASK-2701-add-replication-config-and-s3-target-client.md b/docs/pm/tasks/TASK-2701-add-replication-config-and-s3-target-client.md index 447adb1..4388969 100644 --- a/docs/pm/tasks/TASK-2701-add-replication-config-and-s3-target-client.md +++ b/docs/pm/tasks/TASK-2701-add-replication-config-and-s3-target-client.md @@ -18,7 +18,7 @@ constructed and `enabled` is `false` so the rest of [STORY-0900] short-circuits. ## Files to create / modify - `package.json` — modify (add `@aws-sdk/client-s3`; optionally `@aws-sdk/lib-storage`) -- `libs/nestjs/src/lib/common/config/env.schema.ts` — modify (add `OB_REPLICATION_*` keys) +- `libs/nestjs/src/lib/common/config/env.schema.ts` — modify (add `OPENBUCKET_REPLICATION_*` keys) - `libs/nestjs/src/lib/common/config/app-config.service.ts` — modify (typed getters) - `libs/nestjs/src/lib/open-bucket-options.ts` — modify (`replication?` in options + `ResolvedOpenBucketOptions` + `resolveOptions` defaults + validation) - `libs/nestjs/src/lib/storage/replication/replication-config.ts` — new (`ReplicationConfig` shape + `REPLICATION_CONFIG` token + factory reading either source) @@ -44,18 +44,18 @@ constructed and `enabled` is `false` so the rest of [STORY-0900] short-circuits. } ``` - Env (standalone) — add to `EnvSchema`, all optional so absence ⇒ disabled; - `OB_REPLICATION_ENABLED` gates them and, when true, the endpoint/bucket/creds + `OPENBUCKET_REPLICATION_ENABLED` gates them and, when true, the endpoint/bucket/creds are `.refine`-required together (a partial config must refuse to boot, mirroring the `admin`-block footgun guard in `resolveOptions`): - - `OB_REPLICATION_ENABLED` (bool, default false) - - `OB_REPLICATION_ENDPOINT` (DNS-safe URL; **warn if `http://`** — see security) - - `OB_REPLICATION_REGION` (default 'us-east-1') - - `OB_REPLICATION_BUCKET` - - `OB_REPLICATION_ACCESS_KEY_ID`, `OB_REPLICATION_SECRET_ACCESS_KEY` - - `OB_REPLICATION_FORCE_PATH_STYLE` (bool, default true) - - `OB_REPLICATION_MAX_ATTEMPTS` (int, default 12) - - `OB_REPLICATION_DRAIN_INTERVAL_MS` (int ≥ 1000, default 5000) - - `OB_REPLICATION_BATCH_KEYS` (int, default 50) + - `OPENBUCKET_REPLICATION_ENABLED` (bool, default false) + - `OPENBUCKET_REPLICATION_ENDPOINT` (DNS-safe URL; **warn if `http://`** — see security) + - `OPENBUCKET_REPLICATION_REGION` (default 'us-east-1') + - `OPENBUCKET_REPLICATION_BUCKET` + - `OPENBUCKET_REPLICATION_ACCESS_KEY_ID`, `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` + - `OPENBUCKET_REPLICATION_FORCE_PATH_STYLE` (bool, default true) + - `OPENBUCKET_REPLICATION_MAX_ATTEMPTS` (int, default 12) + - `OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS` (int ≥ 1000, default 5000) + - `OPENBUCKET_REPLICATION_BATCH_KEYS` (int, default 50) - Library `OpenBucketModuleOptions.replication?`: same fields (`credentials: { accessKeyId, secretAccessKey }`). Extend `ResolvedOpenBucketOptions` and apply defaults in `resolveOptions`. The dual-mode `ConfigModule` already chooses env vs options; the @@ -78,7 +78,7 @@ constructed and `enabled` is `false` so the rest of [STORY-0900] short-circuits. double retry loops. - Security / DoS considerations: - **Secrets never logged** — do not log the resolved config; add - `OB_REPLICATION_SECRET_ACCESS_KEY` (and the SDK's `authorization`) to the + `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` (and the SDK's `authorization`) to the pino `redact` paths pattern established in `open-bucket-core.module.ts`. `secretAccessKey` is held only in the `S3Client` credentials closure. - **Plaintext transport** — replicated bytes are the object *plaintext* diff --git a/docs/pm/tasks/TASK-3631-add-scheduled-backup-config-knobs.md b/docs/pm/tasks/TASK-3631-add-scheduled-backup-config-knobs.md index a45a7f9..b178c9a 100644 --- a/docs/pm/tasks/TASK-3631-add-scheduled-backup-config-knobs.md +++ b/docs/pm/tasks/TASK-3631-add-scheduled-backup-config-knobs.md @@ -15,7 +15,7 @@ as replication does (`resolveReplicationConfig` / `REPLICATION_CONFIG`). This is knob surface the runner (TASK-3632) and controller (TASK-3634) read. ## Files to create / modify -- `libs/nestjs/src/lib/common/config/env.schema.ts` — modify (new `OB_SCHEDULED_BACKUP_*` vars + cross-field refine) +- `libs/nestjs/src/lib/common/config/env.schema.ts` — modify (new `OPENBUCKET_SCHEDULED_BACKUP_*` vars + cross-field refine) - `libs/nestjs/src/lib/common/config/app-config.service.ts` — modify (typed accessors) - `libs/nestjs/src/lib/open-bucket-options.ts` — modify (`backups?` block on `OpenBucketModuleOptions` + `ResolvedOpenBucketOptions`, `resolveOptions`, `validateSecurityCriticalOptions`) - `libs/nestjs/src/lib/common/config/config-source.ts` — modify (map resolved options → env-shaped config, apply numeric defaults) @@ -23,20 +23,20 @@ knob surface the runner (TASK-3632) and controller (TASK-3634) read. - `libs/nestjs/package.json`, `apps/openbucket-backend/package.json`, root `package.json` — modify (add `cron-parser`) ## Implementation notes -- Env vars (mirror the `OB_REPLICATION_*` block, `z.coerce.number().int().min().max().default()`): - - `OB_SCHEDULED_BACKUP_ENABLED` — `envBoolean(false)` - - `OB_SCHEDULED_BACKUP_SCOPE` — `z.enum(['instance','buckets']).default('instance')` (`buckets` = one snapshot per bucket) - - `OB_SCHEDULED_BACKUP_INTERVAL_MINUTES` — `z.coerce.number().int().min(5).max(43200).optional()` - - `OB_SCHEDULED_BACKUP_CRON` — `z.string().optional()` (5-field cron; validated below) - - `OB_SCHEDULED_BACKUP_DIR` — `z.string().optional()` (default `/backups` applied at resolve time) - - `OB_SCHEDULED_BACKUP_KEEP_LAST` — `z.coerce.number().int().min(1).max(1000).default(7)` - - `OB_SCHEDULED_BACKUP_MAX_AGE_DAYS` — `z.coerce.number().int().min(1).max(3650).default(30)` - - `OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS` — `z.coerce.number().int().min(10_000).max(3_600_000).default(60_000)` (the fixed wake tick) - - `OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION` — `envBoolean(false)` -- Cross-field `superRefine` when `OB_SCHEDULED_BACKUP_ENABLED`: +- Env vars (mirror the `OPENBUCKET_REPLICATION_*` block, `z.coerce.number().int().min().max().default()`): + - `OPENBUCKET_SCHEDULED_BACKUP_ENABLED` — `envBoolean(false)` + - `OPENBUCKET_SCHEDULED_BACKUP_SCOPE` — `z.enum(['instance','buckets']).default('instance')` (`buckets` = one snapshot per bucket) + - `OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES` — `z.coerce.number().int().min(5).max(43200).optional()` + - `OPENBUCKET_SCHEDULED_BACKUP_CRON` — `z.string().optional()` (5-field cron; validated below) + - `OPENBUCKET_SCHEDULED_BACKUP_DIR` — `z.string().optional()` (default `/backups` applied at resolve time) + - `OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST` — `z.coerce.number().int().min(1).max(1000).default(7)` + - `OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS` — `z.coerce.number().int().min(1).max(3650).default(30)` + - `OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS` — `z.coerce.number().int().min(10_000).max(3_600_000).default(60_000)` (the fixed wake tick) + - `OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION` — `envBoolean(false)` +- Cross-field `superRefine` when `OPENBUCKET_SCHEDULED_BACKUP_ENABLED`: - exactly one of `INTERVAL_MINUTES` / `CRON` must be set (mutually exclusive; error otherwise); - if `CRON` is set, parse it with `cron-parser` (`CronExpressionParser.parse(cron)` in a try/catch) and add a Zod issue on failure — **fail fast at boot**, never mid-tick; - - if `PUSH_TO_REPLICATION` is true but `OB_REPLICATION_ENABLED` is false, do **not** hard-fail — log a boot WARNING (the flag is a no-op) so an operator toggling replication later isn't blocked. + - if `PUSH_TO_REPLICATION` is true but `OPENBUCKET_REPLICATION_ENABLED` is false, do **not** hard-fail — log a boot WARNING (the flag is a no-op) so an operator toggling replication later isn't blocked. - `ScheduledBackupConfig` (resolved shape both sources funnel through), plus a `DISABLED` const like `replication-config.ts`: @@ -76,8 +76,8 @@ knob surface the runner (TASK-3632) and controller (TASK-3634) read. hostile-tiny value can't busy-loop the scheduler. ## Acceptance criteria -- [ ] `OB_SCHEDULED_BACKUP_*` vars parse with defaults; enabling with neither/both of interval+cron fails boot with a clear message. -- [ ] A malformed `OB_SCHEDULED_BACKUP_CRON` fails `env.schema` parsing (and `validateSecurityCriticalOptions` for the library path). +- [ ] `OPENBUCKET_SCHEDULED_BACKUP_*` vars parse with defaults; enabling with neither/both of interval+cron fails boot with a clear message. +- [ ] A malformed `OPENBUCKET_SCHEDULED_BACKUP_CRON` fails `env.schema` parsing (and `validateSecurityCriticalOptions` for the library path). - [ ] `resolveScheduledBackupConfig` returns `{ enabled: false, … }` when unset and a fully-defaulted shape when enabled; `dir` defaults to `/backups`. - [ ] `cron-parser` appears in all three package.json files and is `external` in the backend webpack bundle (`nx build openbucket-backend` succeeds, bundle does not inline it). - [ ] `nx test nestjs --testPathPattern='env.schema|open-bucket-options'` passes with the new cases. diff --git a/docs/pm/tasks/TASK-3642-integrity-scrub-runner.md b/docs/pm/tasks/TASK-3642-integrity-scrub-runner.md index 24ba6b3..8bc319f 100644 --- a/docs/pm/tasks/TASK-3642-integrity-scrub-runner.md +++ b/docs/pm/tasks/TASK-3642-integrity-scrub-runner.md @@ -61,11 +61,11 @@ follows the `TieringSweepRunner` / `ReconcileRunner` throttling shape exactly. the admin `status` endpoint reads durable numbers. When `scanForScrub` returns empty, reset the cursor to null (full pass complete) and stamp `lastRunAt`. - Config knobs (`env.schema.ts`, follow the `envBoolean` / `z.coerce.number` pattern used by - `USAGE_ROLLUP_INTERVAL_MS` and `OB_REPLICATION_ENABLED`): - - `OB_INTEGRITY_SCRUB_ENABLED` → `envBoolean(false)` - - `OB_INTEGRITY_SCRUB_INTERVAL_MS` → `z.coerce.number().int().min(1_000).default(60_000)` - - `OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` → `z.coerce.number().int().min(1).default(1000)` - - `OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` → `z.coerce.number().int().positive().default(1_073_741_824)` (1 GiB/tick throttle) + `USAGE_ROLLUP_INTERVAL_MS` and `OPENBUCKET_REPLICATION_ENABLED`): + - `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` → `envBoolean(false)` + - `OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS` → `z.coerce.number().int().min(1_000).default(60_000)` + - `OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` → `z.coerce.number().int().min(1).default(1000)` + - `OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` → `z.coerce.number().int().positive().default(1_073_741_824)` (1 GiB/tick throttle) - Registration: add `IntegrityScrubRunner` BOTH to `providers` and to the `SCHEDULED_TASKS` factory `inject` array in `background.module.ts` (NestJS has no `multi` flag — see the module's own doc comment). @@ -80,7 +80,7 @@ follows the `TieringSweepRunner` / `ReconcileRunner` throttling shape exactly. configured target coordinates, bounded to 255). The byte budget bounds disk read amplification. ## Acceptance criteria -- [ ] With `OB_INTEGRITY_SCRUB_ENABLED` unset, `run()` returns before any repository/blob access (asserted via mocks). +- [ ] With `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` unset, `run()` returns before any repository/blob access (asserted via mocks). - [ ] A tick hashing enough to hit `MAX_OBJECTS_PER_TICK` or `MAX_BYTES_PER_TICK` persists the cursor and stops; the next tick resumes from it. - [ ] A blob whose bytes are flipped on disk is marked `integrityStatus='corrupt'` with `integrityCheckedAt` set; an intact blob is marked `ok`. - [ ] `IntegrityScrubRunner` appears in `background.module.ts` providers and the `SCHEDULED_TASKS` inject list; `nx test nestjs --testPathPattern=integrity-scrub.runner` passes. diff --git a/docs/pm/test-plans/TEST-1203-scheduled-backups.md b/docs/pm/test-plans/TEST-1203-scheduled-backups.md index 8ceb0c4..954702c 100644 --- a/docs/pm/test-plans/TEST-1203-scheduled-backups.md +++ b/docs/pm/test-plans/TEST-1203-scheduled-backups.md @@ -18,7 +18,7 @@ leaking paths, keys, or secrets. - Jest unit/integration harness (as `scheduled-backup.*.spec.ts`, `backup.service.spec.ts`). - A fake `Clock` (the injectable `Clock` used by `trash-purge` / `usage-rollup`) so retention windows and cron/interval due-times are fast-forwarded deterministically. -- A temp `DATA_DIR` per test; `OB_SCHEDULED_BACKUP_DIR` under it (or defaulted). +- A temp `DATA_DIR` per test; `OPENBUCKET_SCHEDULED_BACKUP_DIR` under it (or defaulted). - A stub `ReplicationTargetService` capturing `putObject({ key, contentLength, contentType })` and toggleable to throw, plus a `.enabled` toggle. - Seed a bucket + a few objects via the domain services so `writeSnapshot` has real @@ -28,7 +28,7 @@ leaking paths, keys, or secrets. ## Cases 1. **Snapshot parity (TASK-3630)** — given seeded objects, when `writeSnapshot(sink, 'instance', names)` writes to a file sink, then the resulting `.zip`'s `manifest.json` + `data/…` entries are byte-identical to `streamInstanceBackup`'s download, and the returned `{ bytes, objectCount }` matches the archive. -2. **Config resolution + fail-fast (TASK-3631)** — given `OB_SCHEDULED_BACKUP_ENABLED=true` with neither interval nor cron, boot fails; with both set, boot fails; with a malformed `OB_SCHEDULED_BACKUP_CRON`, boot fails; with a valid cron only, `resolveScheduledBackupConfig` returns the defaulted shape and `dir` = `/backups`. +2. **Config resolution + fail-fast (TASK-3631)** — given `OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true` with neither interval nor cron, boot fails; with both set, boot fails; with a malformed `OPENBUCKET_SCHEDULED_BACKUP_CRON`, boot fails; with a valid cron only, `resolveScheduledBackupConfig` returns the defaulted shape and `dir` = `/backups`. 3. **Disabled = no-op (TASK-3632)** — given `enabled: false`, when the runner ticks, then no file is written and `state.json` is absent. 4. **Interval due + atomic write (TASK-3632)** — given `intervalMinutes: 60` and last-run 61 min ago (fake clock), when the runner ticks, then one `.zip` + `.json` sidecar exist, the file mode is `0o600`, no `.part` remains, and `state.json.lastStatus === 'ok'` with a fresh `lastRunAt`; ticking again 1 min later writes nothing (not yet due). 5. **Per-bucket scope + failure isolation (TASK-3632/3633)** — given `scope: 'buckets'` with 3 buckets where one object read throws, when a cycle runs, then the other two buckets produce snapshots and the failure is logged (cycle not aborted); retention is applied per bucket. diff --git a/docs/pm/test-plans/TEST-1204-integrity-scrub.md b/docs/pm/test-plans/TEST-1204-integrity-scrub.md index 1f6b27b..85033ba 100644 --- a/docs/pm/test-plans/TEST-1204-integrity-scrub.md +++ b/docs/pm/test-plans/TEST-1204-integrity-scrub.md @@ -23,11 +23,11 @@ through the admin API and console indicator — all without leaking any target c - Replication: a fake S3 target (e.g. an in-process `@aws-sdk/client-s3` mock or a local S3 double) preloaded with the GOOD copy of the object under its raw key; plus a run with the target disabled and a run where the remote copy is ALSO corrupt. -- Env toggles: `OB_INTEGRITY_SCRUB_ENABLED`, `OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK`, - `OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK`, `OB_REPLICATION_ENABLED`. +- Env toggles: `OPENBUCKET_INTEGRITY_SCRUB_ENABLED`, `OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK`, + `OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK`, `OPENBUCKET_REPLICATION_ENABLED`. ## Cases -1. Default-off: with `OB_INTEGRITY_SCRUB_ENABLED` unset, invoking `run()` performs zero +1. Default-off: with `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` unset, invoking `run()` performs zero `scanForScrub`/`getBlob` calls (asserted via spies) and writes nothing to the DB. 2. Verifier ok/corrupt/SSE: `IntegrityVerifier.verify` returns `ok:true` for an intact blob, `ok:false` with the recomputed digest for a byte-flipped blob, and correctly decrypts + diff --git a/libs/nestjs/README.md b/libs/nestjs/README.md index b2176c1..5506bad 100644 --- a/libs/nestjs/README.md +++ b/libs/nestjs/README.md @@ -813,7 +813,7 @@ OpenBucketModule.forRoot({ }) ``` -Standalone (env-configured) deployments use the equivalent `OB_REPLICATION_*` +Standalone (env-configured) deployments use the equivalent `OPENBUCKET_REPLICATION_*` variables — see the [root README](../../README.md#async-replication). - **A present-but-partial `replication` block refuses to boot** (you must supply @@ -909,7 +909,7 @@ OpenAPI doc so the generated client has a typed `BackupScheduleService`): The admin console's **Settings → Backup & Restore** tab shows last-run / next-run + a snapshot count and a **Run now** button. -Standalone (env) equivalents: `OB_SCHEDULED_BACKUP_ENABLED`, `_SCOPE`, +Standalone (env) equivalents: `OPENBUCKET_SCHEDULED_BACKUP_ENABLED`, `_SCOPE`, `_INTERVAL_MINUTES` / `_CRON`, `_DIR`, `_KEEP_LAST`, `_MAX_AGE_DAYS`, `_CHECK_INTERVAL_MS`, `_PUSH_TO_REPLICATION` — see `.env.example`. @@ -998,15 +998,15 @@ between batches (the same throttling shape as the tiering/reconcile runners). Tiered objects (`location !== 'local'`) and pre-F1 rows without a stored `contentSha256` are skipped, never marked corrupt. -Standalone deployments configure it via `OB_INTEGRITY_SCRUB_*` environment +Standalone deployments configure it via `OPENBUCKET_INTEGRITY_SCRUB_*` environment variables (defaults shown): | Variable | Default | Purpose | | --- | --- | --- | -| `OB_INTEGRITY_SCRUB_ENABLED` | `false` | Master switch. A fresh install performs zero extra disk reads / DB writes. | -| `OB_INTEGRITY_SCRUB_INTERVAL_MS` | `60000` | Tick interval (floor 1s). | -| `OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` | `1000` | Hard per-tick object cap — bounds detection work regardless of blob sizes. | -| `OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` | `1073741824` (1 GiB) | Per-tick byte budget: the tick stops once this many bytes have been hashed. | +| `OPENBUCKET_INTEGRITY_SCRUB_ENABLED` | `false` | Master switch. A fresh install performs zero extra disk reads / DB writes. | +| `OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS` | `60000` | Tick interval (floor 1s). | +| `OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK` | `1000` | Hard per-tick object cap — bounds detection work regardless of blob sizes. | +| `OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK` | `1073741824` (1 GiB) | Per-tick byte budget: the tick stops once this many bytes have been hashed. | The admin API exposes a read model + a manual trigger under `/api/admin/integrity` (JWT-guarded, in the OpenAPI doc so the generated client has a typed diff --git a/libs/nestjs/src/lib/admin/backup/scheduled-backup-config.ts b/libs/nestjs/src/lib/admin/backup/scheduled-backup-config.ts index 8c51798..0e7aeb5 100644 --- a/libs/nestjs/src/lib/admin/backup/scheduled-backup-config.ts +++ b/libs/nestjs/src/lib/admin/backup/scheduled-backup-config.ts @@ -66,9 +66,9 @@ export function resolveScheduledBackupConfig(config: AppConfigService): Schedule if (config.scheduledBackupPushToReplication && !config.replicationEnabled) { new Logger('ScheduledBackupConfig').warn( - 'OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true but replication is disabled — ' + + 'OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION=true but replication is disabled — ' + 'snapshots will be written locally only (the push is a no-op). Enable ' + - 'OB_REPLICATION_* to push snapshots off-box.', + 'OPENBUCKET_REPLICATION_* to push snapshots off-box.', ); } diff --git a/libs/nestjs/src/lib/common/config/app-config.service.ts b/libs/nestjs/src/lib/common/config/app-config.service.ts index 268821e..7290047 100644 --- a/libs/nestjs/src/lib/common/config/app-config.service.ts +++ b/libs/nestjs/src/lib/common/config/app-config.service.ts @@ -81,28 +81,28 @@ export class AppConfigService { } // --- async replication to external S3 target (STORY-0900) --- - get replicationEnabled(): boolean { return this.raw.get('OB_REPLICATION_ENABLED', { infer: true }); } - get replicationEndpoint(): string | undefined { return this.raw.get('OB_REPLICATION_ENDPOINT', { infer: true }); } - get replicationRegion(): string { return this.raw.get('OB_REPLICATION_REGION', { infer: true }); } - get replicationBucket(): string | undefined { return this.raw.get('OB_REPLICATION_BUCKET', { infer: true }); } - get replicationAccessKeyId(): string | undefined { return this.raw.get('OB_REPLICATION_ACCESS_KEY_ID', { infer: true }); } - get replicationSecretAccessKey(): string | undefined { return this.raw.get('OB_REPLICATION_SECRET_ACCESS_KEY', { infer: true }); } - get replicationForcePathStyle(): boolean { return this.raw.get('OB_REPLICATION_FORCE_PATH_STYLE', { infer: true }); } - get replicationMaxAttempts(): number { return this.raw.get('OB_REPLICATION_MAX_ATTEMPTS', { infer: true }); } - get replicationDrainIntervalMs(): number { return this.raw.get('OB_REPLICATION_DRAIN_INTERVAL_MS', { infer: true }); } - get replicationBatchKeys(): number { return this.raw.get('OB_REPLICATION_BATCH_KEYS', { infer: true }); } - get replicationLargeObjectThresholdBytes(): number { return this.raw.get('OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES', { infer: true }); } + get replicationEnabled(): boolean { return this.raw.get('OPENBUCKET_REPLICATION_ENABLED', { infer: true }); } + get replicationEndpoint(): string | undefined { return this.raw.get('OPENBUCKET_REPLICATION_ENDPOINT', { infer: true }); } + get replicationRegion(): string { return this.raw.get('OPENBUCKET_REPLICATION_REGION', { infer: true }); } + get replicationBucket(): string | undefined { return this.raw.get('OPENBUCKET_REPLICATION_BUCKET', { infer: true }); } + get replicationAccessKeyId(): string | undefined { return this.raw.get('OPENBUCKET_REPLICATION_ACCESS_KEY_ID', { infer: true }); } + get replicationSecretAccessKey(): string | undefined { return this.raw.get('OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY', { infer: true }); } + get replicationForcePathStyle(): boolean { return this.raw.get('OPENBUCKET_REPLICATION_FORCE_PATH_STYLE', { infer: true }); } + get replicationMaxAttempts(): number { return this.raw.get('OPENBUCKET_REPLICATION_MAX_ATTEMPTS', { infer: true }); } + get replicationDrainIntervalMs(): number { return this.raw.get('OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS', { infer: true }); } + get replicationBatchKeys(): number { return this.raw.get('OPENBUCKET_REPLICATION_BATCH_KEYS', { infer: true }); } + get replicationLargeObjectThresholdBytes(): number { return this.raw.get('OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES', { infer: true }); } // --- scheduled backups & retention (STORY-1203) --- - get scheduledBackupEnabled(): boolean { return this.raw.get('OB_SCHEDULED_BACKUP_ENABLED', { infer: true }); } - get scheduledBackupScope(): Env['OB_SCHEDULED_BACKUP_SCOPE'] { return this.raw.get('OB_SCHEDULED_BACKUP_SCOPE', { infer: true }); } - get scheduledBackupIntervalMinutes(): number | undefined { return this.raw.get('OB_SCHEDULED_BACKUP_INTERVAL_MINUTES', { infer: true }); } - get scheduledBackupCron(): string | undefined { return this.raw.get('OB_SCHEDULED_BACKUP_CRON', { infer: true }); } - get scheduledBackupDir(): string | undefined { return this.raw.get('OB_SCHEDULED_BACKUP_DIR', { infer: true }); } - get scheduledBackupKeepLast(): number { return this.raw.get('OB_SCHEDULED_BACKUP_KEEP_LAST', { infer: true }); } - get scheduledBackupMaxAgeDays(): number { return this.raw.get('OB_SCHEDULED_BACKUP_MAX_AGE_DAYS', { infer: true }); } - get scheduledBackupCheckIntervalMs(): number { return this.raw.get('OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS', { infer: true }); } - get scheduledBackupPushToReplication(): boolean { return this.raw.get('OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION', { infer: true }); } + get scheduledBackupEnabled(): boolean { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_ENABLED', { infer: true }); } + get scheduledBackupScope(): Env['OPENBUCKET_SCHEDULED_BACKUP_SCOPE'] { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_SCOPE', { infer: true }); } + get scheduledBackupIntervalMinutes(): number | undefined { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES', { infer: true }); } + get scheduledBackupCron(): string | undefined { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_CRON', { infer: true }); } + get scheduledBackupDir(): string | undefined { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_DIR', { infer: true }); } + get scheduledBackupKeepLast(): number { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST', { infer: true }); } + get scheduledBackupMaxAgeDays(): number { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS', { infer: true }); } + get scheduledBackupCheckIntervalMs(): number { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS', { infer: true }); } + get scheduledBackupPushToReplication(): boolean { return this.raw.get('OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION', { infer: true }); } // --- Prometheus /metrics + OpenTelemetry (STORY-1202) --- get metricsMode(): Env['METRICS_MODE'] { return this.raw.get('METRICS_MODE', { infer: true }); } @@ -117,8 +117,8 @@ export class AppConfigService { get tierPresignTtlSeconds(): number { return this.raw.get('OPENBUCKET_TIER_PRESIGN_TTL_SECONDS', { infer: true }); } // --- background integrity scrubbing (STORY-1204) --- - get integrityScrubEnabled(): boolean { return this.raw.get('OB_INTEGRITY_SCRUB_ENABLED', { infer: true }); } - get integrityScrubIntervalMs(): number { return this.raw.get('OB_INTEGRITY_SCRUB_INTERVAL_MS', { infer: true }); } - get integrityScrubMaxObjectsPerTick(): number { return this.raw.get('OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK', { infer: true }); } - get integrityScrubMaxBytesPerTick(): number { return this.raw.get('OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK', { infer: true }); } + get integrityScrubEnabled(): boolean { return this.raw.get('OPENBUCKET_INTEGRITY_SCRUB_ENABLED', { infer: true }); } + get integrityScrubIntervalMs(): number { return this.raw.get('OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS', { infer: true }); } + get integrityScrubMaxObjectsPerTick(): number { return this.raw.get('OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK', { infer: true }); } + get integrityScrubMaxBytesPerTick(): number { return this.raw.get('OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK', { infer: true }); } } diff --git a/libs/nestjs/src/lib/common/config/config-source.ts b/libs/nestjs/src/lib/common/config/config-source.ts index 1c05159..a81e62f 100644 --- a/libs/nestjs/src/lib/common/config/config-source.ts +++ b/libs/nestjs/src/lib/common/config/config-source.ts @@ -95,32 +95,32 @@ export function buildConfig(opts?: ResolvedOpenBucketOptions): Env { // Async replication (STORY-0900). Off unless the host passes a `replication` // block; resolved defaults mirror the env schema. `enabled` is derived from // the block's presence (the library caller never sets it explicitly). - OB_REPLICATION_ENABLED: !!opts.replication, - OB_REPLICATION_ENDPOINT: opts.replication?.endpoint, - OB_REPLICATION_REGION: opts.replication?.region ?? 'us-east-1', - OB_REPLICATION_BUCKET: opts.replication?.bucket, - OB_REPLICATION_ACCESS_KEY_ID: opts.replication?.credentials.accessKeyId, - OB_REPLICATION_SECRET_ACCESS_KEY: opts.replication?.credentials.secretAccessKey, - OB_REPLICATION_FORCE_PATH_STYLE: opts.replication?.forcePathStyle ?? true, - OB_REPLICATION_MAX_ATTEMPTS: opts.replication?.maxAttempts ?? 12, - OB_REPLICATION_DRAIN_INTERVAL_MS: opts.replication?.drainIntervalMs ?? 5_000, - OB_REPLICATION_BATCH_KEYS: opts.replication?.batchKeys ?? 50, - OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES: + OPENBUCKET_REPLICATION_ENABLED: !!opts.replication, + OPENBUCKET_REPLICATION_ENDPOINT: opts.replication?.endpoint, + OPENBUCKET_REPLICATION_REGION: opts.replication?.region ?? 'us-east-1', + OPENBUCKET_REPLICATION_BUCKET: opts.replication?.bucket, + OPENBUCKET_REPLICATION_ACCESS_KEY_ID: opts.replication?.credentials.accessKeyId, + OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY: opts.replication?.credentials.secretAccessKey, + OPENBUCKET_REPLICATION_FORCE_PATH_STYLE: opts.replication?.forcePathStyle ?? true, + OPENBUCKET_REPLICATION_MAX_ATTEMPTS: opts.replication?.maxAttempts ?? 12, + OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS: opts.replication?.drainIntervalMs ?? 5_000, + OPENBUCKET_REPLICATION_BATCH_KEYS: opts.replication?.batchKeys ?? 50, + OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES: opts.replication?.largeObjectThresholdBytes ?? 64 * 1024 * 1024, // Scheduled backups (STORY-1203). Off unless the host passes a `backups` // block; `enabled` is derived from its presence (the library caller never // sets it explicitly). Numeric defaults mirror the env schema; the default // `dir` (`/backups`) is applied at resolve time in // `resolveScheduledBackupConfig`, so an unset `dir` maps to `undefined` here. - OB_SCHEDULED_BACKUP_ENABLED: !!opts.backups, - OB_SCHEDULED_BACKUP_SCOPE: opts.backups?.scope ?? 'instance', - OB_SCHEDULED_BACKUP_INTERVAL_MINUTES: opts.backups?.intervalMinutes, - OB_SCHEDULED_BACKUP_CRON: opts.backups?.cron, - OB_SCHEDULED_BACKUP_DIR: opts.backups?.dir, - OB_SCHEDULED_BACKUP_KEEP_LAST: opts.backups?.keepLast ?? 7, - OB_SCHEDULED_BACKUP_MAX_AGE_DAYS: opts.backups?.maxAgeDays ?? 30, - OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS: opts.backups?.checkIntervalMs ?? 60_000, - OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION: opts.backups?.pushToReplication ?? false, + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: !!opts.backups, + OPENBUCKET_SCHEDULED_BACKUP_SCOPE: opts.backups?.scope ?? 'instance', + OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES: opts.backups?.intervalMinutes, + OPENBUCKET_SCHEDULED_BACKUP_CRON: opts.backups?.cron, + OPENBUCKET_SCHEDULED_BACKUP_DIR: opts.backups?.dir, + OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST: opts.backups?.keepLast ?? 7, + OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS: opts.backups?.maxAgeDays ?? 30, + OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS: opts.backups?.checkIntervalMs ?? 60_000, + OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION: opts.backups?.pushToReplication ?? false, // Cold-object tiering (STORY-0901). Off by default; a library host enables it // via the env-driven standalone path. Defaults mirror the env schema. OPENBUCKET_TIER_ENABLED: false, @@ -130,10 +130,10 @@ export function buildConfig(opts?: ResolvedOpenBucketOptions): Env { OPENBUCKET_TIER_PRESIGN_TTL_SECONDS: 300, // Background integrity scrubbing (STORY-1204). Off by default; a library host // enables it via the env-driven standalone path. Defaults mirror the env schema. - OB_INTEGRITY_SCRUB_ENABLED: false, - OB_INTEGRITY_SCRUB_INTERVAL_MS: 60_000, - OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK: 1_000, - OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK: 1_073_741_824, + OPENBUCKET_INTEGRITY_SCRUB_ENABLED: false, + OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS: 60_000, + OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK: 1_000, + OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK: 1_073_741_824, // Prometheus /metrics + OpenTelemetry (STORY-1202). Off by default; a library // host opts in via the `metrics` / `tracing` option blocks. `resolveOptions` // already defaulted the mode to 'off'. diff --git a/libs/nestjs/src/lib/common/config/env.schema.spec.ts b/libs/nestjs/src/lib/common/config/env.schema.spec.ts index 1d340e5..942bdc0 100644 --- a/libs/nestjs/src/lib/common/config/env.schema.spec.ts +++ b/libs/nestjs/src/lib/common/config/env.schema.spec.ts @@ -253,63 +253,63 @@ describe('loadEnv', () => { // --- async replication to external S3 target (STORY-0900) --- it('case 22: replication off by default with documented drain/dead-letter defaults', () => { const env = loadEnv({ ...baseEnv }); - expect(env.OB_REPLICATION_ENABLED).toBe(false); - expect(env.OB_REPLICATION_REGION).toBe('us-east-1'); - expect(env.OB_REPLICATION_FORCE_PATH_STYLE).toBe(true); - expect(env.OB_REPLICATION_MAX_ATTEMPTS).toBe(12); - expect(env.OB_REPLICATION_DRAIN_INTERVAL_MS).toBe(5_000); - expect(env.OB_REPLICATION_BATCH_KEYS).toBe(50); - expect(env.OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES).toBe(64 * 1024 * 1024); + expect(env.OPENBUCKET_REPLICATION_ENABLED).toBe(false); + expect(env.OPENBUCKET_REPLICATION_REGION).toBe('us-east-1'); + expect(env.OPENBUCKET_REPLICATION_FORCE_PATH_STYLE).toBe(true); + expect(env.OPENBUCKET_REPLICATION_MAX_ATTEMPTS).toBe(12); + expect(env.OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS).toBe(5_000); + expect(env.OPENBUCKET_REPLICATION_BATCH_KEYS).toBe(50); + expect(env.OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES).toBe(64 * 1024 * 1024); }); it('case 23: a full replication config boots and enables replication', () => { const env = loadEnv({ ...baseEnv, - OB_REPLICATION_ENABLED: 'true', - OB_REPLICATION_ENDPOINT: 'https://s3.example.com', - OB_REPLICATION_BUCKET: 'remote-mirror', - OB_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', - OB_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', + OPENBUCKET_REPLICATION_ENABLED: 'true', + OPENBUCKET_REPLICATION_ENDPOINT: 'https://s3.example.com', + OPENBUCKET_REPLICATION_BUCKET: 'remote-mirror', + OPENBUCKET_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', + OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', }); - expect(env.OB_REPLICATION_ENABLED).toBe(true); - expect(env.OB_REPLICATION_BUCKET).toBe('remote-mirror'); + expect(env.OPENBUCKET_REPLICATION_ENABLED).toBe(true); + expect(env.OPENBUCKET_REPLICATION_BUCKET).toBe('remote-mirror'); }); it('case 24: ENABLED=true but missing bucket/creds refuses to boot (fail-closed)', () => { - expect(() => loadEnv({ ...baseEnv, OB_REPLICATION_ENABLED: 'true' })).toThrow( + expect(() => loadEnv({ ...baseEnv, OPENBUCKET_REPLICATION_ENABLED: 'true' })).toThrow( 'Refusing to boot: invalid environment.', ); const joined = errSpy.mock.calls[0][0] as string; - expect(joined).toContain('OB_REPLICATION_BUCKET'); - expect(joined).toContain('OB_REPLICATION_ACCESS_KEY_ID'); - expect(joined).toContain('OB_REPLICATION_SECRET_ACCESS_KEY'); + expect(joined).toContain('OPENBUCKET_REPLICATION_BUCKET'); + expect(joined).toContain('OPENBUCKET_REPLICATION_ACCESS_KEY_ID'); + expect(joined).toContain('OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY'); }); it('case 25: ENABLED=true with a malformed remote bucket name is rejected', () => { expect(() => loadEnv({ ...baseEnv, - OB_REPLICATION_ENABLED: 'true', - OB_REPLICATION_BUCKET: 'Invalid_Bucket_NAME', - OB_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', - OB_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', + OPENBUCKET_REPLICATION_ENABLED: 'true', + OPENBUCKET_REPLICATION_BUCKET: 'Invalid_Bucket_NAME', + OPENBUCKET_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', + OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', }), ).toThrow('Refusing to boot: invalid environment.'); - expect(errSpy.mock.calls[0][0]).toContain('OB_REPLICATION_BUCKET must be a valid S3 bucket name'); + expect(errSpy.mock.calls[0][0]).toContain('OPENBUCKET_REPLICATION_BUCKET must be a valid S3 bucket name'); }); it('case 26: ENABLED=true with a malformed endpoint URL is rejected', () => { expect(() => loadEnv({ ...baseEnv, - OB_REPLICATION_ENABLED: 'true', - OB_REPLICATION_ENDPOINT: 'not-a-url', - OB_REPLICATION_BUCKET: 'remote-mirror', - OB_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', - OB_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', + OPENBUCKET_REPLICATION_ENABLED: 'true', + OPENBUCKET_REPLICATION_ENDPOINT: 'not-a-url', + OPENBUCKET_REPLICATION_BUCKET: 'remote-mirror', + OPENBUCKET_REPLICATION_ACCESS_KEY_ID: 'AKIAEXAMPLE1234567890', + OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY: 'k7Jf2pQrwStN9vB3zX1cM4dL0eR6yU2h7gK3nP5s', }), ).toThrow('Refusing to boot: invalid environment.'); - expect(errSpy.mock.calls[0][0]).toContain('OB_REPLICATION_ENDPOINT'); + expect(errSpy.mock.calls[0][0]).toContain('OPENBUCKET_REPLICATION_ENDPOINT'); }); it('case 27: cold-object tiering knobs apply their documented defaults (STORY-0901)', () => { @@ -449,47 +449,47 @@ describe('loadEnv scheduled backups (STORY-1203)', () => { it('defaults: disabled with the documented retention defaults', () => { const env = loadEnv({ ...baseEnv }); - expect(env.OB_SCHEDULED_BACKUP_ENABLED).toBe(false); - expect(env.OB_SCHEDULED_BACKUP_SCOPE).toBe('instance'); - expect(env.OB_SCHEDULED_BACKUP_KEEP_LAST).toBe(7); - expect(env.OB_SCHEDULED_BACKUP_MAX_AGE_DAYS).toBe(30); - expect(env.OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS).toBe(60_000); - expect(env.OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION).toBe(false); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_ENABLED).toBe(false); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_SCOPE).toBe('instance'); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST).toBe(7); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS).toBe(30); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS).toBe(60_000); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION).toBe(false); }); it('enabled with an interval parses', () => { const env = loadEnv({ ...baseEnv, - OB_SCHEDULED_BACKUP_ENABLED: 'true', - OB_SCHEDULED_BACKUP_INTERVAL_MINUTES: '60', + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: 'true', + OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES: '60', }); - expect(env.OB_SCHEDULED_BACKUP_ENABLED).toBe(true); - expect(env.OB_SCHEDULED_BACKUP_INTERVAL_MINUTES).toBe(60); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_ENABLED).toBe(true); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES).toBe(60); }); it('enabled with a valid cron parses', () => { const env = loadEnv({ ...baseEnv, - OB_SCHEDULED_BACKUP_ENABLED: 'true', - OB_SCHEDULED_BACKUP_CRON: '0 3 * * *', + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: 'true', + OPENBUCKET_SCHEDULED_BACKUP_CRON: '0 3 * * *', }); - expect(env.OB_SCHEDULED_BACKUP_CRON).toBe('0 3 * * *'); + expect(env.OPENBUCKET_SCHEDULED_BACKUP_CRON).toBe('0 3 * * *'); }); it('enabled with NEITHER interval nor cron refuses to boot', () => { - expect(() => loadEnv({ ...baseEnv, OB_SCHEDULED_BACKUP_ENABLED: 'true' })).toThrow( + expect(() => loadEnv({ ...baseEnv, OPENBUCKET_SCHEDULED_BACKUP_ENABLED: 'true' })).toThrow( 'Refusing to boot: invalid environment.', ); - expect(errSpy.mock.calls[0][0]).toContain('exactly one of OB_SCHEDULED_BACKUP_INTERVAL_MINUTES'); + expect(errSpy.mock.calls[0][0]).toContain('exactly one of OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES'); }); it('enabled with BOTH interval and cron refuses to boot', () => { expect(() => loadEnv({ ...baseEnv, - OB_SCHEDULED_BACKUP_ENABLED: 'true', - OB_SCHEDULED_BACKUP_INTERVAL_MINUTES: '60', - OB_SCHEDULED_BACKUP_CRON: '0 3 * * *', + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: 'true', + OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES: '60', + OPENBUCKET_SCHEDULED_BACKUP_CRON: '0 3 * * *', }), ).toThrow('Refusing to boot: invalid environment.'); }); @@ -498,11 +498,11 @@ describe('loadEnv scheduled backups (STORY-1203)', () => { expect(() => loadEnv({ ...baseEnv, - OB_SCHEDULED_BACKUP_ENABLED: 'true', - OB_SCHEDULED_BACKUP_CRON: 'not a cron', + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: 'true', + OPENBUCKET_SCHEDULED_BACKUP_CRON: 'not a cron', }), ).toThrow('Refusing to boot: invalid environment.'); - expect(errSpy.mock.calls[0][0]).toContain('OB_SCHEDULED_BACKUP_CRON is not a valid cron'); + expect(errSpy.mock.calls[0][0]).toContain('OPENBUCKET_SCHEDULED_BACKUP_CRON is not a valid cron'); }); }); diff --git a/libs/nestjs/src/lib/common/config/env.schema.ts b/libs/nestjs/src/lib/common/config/env.schema.ts index 7e598f5..5a93f39 100644 --- a/libs/nestjs/src/lib/common/config/env.schema.ts +++ b/libs/nestjs/src/lib/common/config/env.schema.ts @@ -110,7 +110,7 @@ export const validateCronExpression = (cron: string): string | null => { CronExpressionParser.parse(cron); return null; } catch (err) { - return `OB_SCHEDULED_BACKUP_CRON is not a valid cron expression: ${(err as Error).message}`; + return `OPENBUCKET_SCHEDULED_BACKUP_CRON is not a valid cron expression: ${(err as Error).message}`; } }; @@ -142,10 +142,10 @@ export const validateReplicationEndpoint = ( try { parsed = new URL(endpoint); } catch { - return { error: 'OB_REPLICATION_ENDPOINT must be a valid URL' }; + return { error: 'OPENBUCKET_REPLICATION_ENDPOINT must be a valid URL' }; } if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { - return { error: 'OB_REPLICATION_ENDPOINT must be an http(s) URL' }; + return { error: 'OPENBUCKET_REPLICATION_ENDPOINT must be an http(s) URL' }; } return { insecure: parsed.protocol === 'http:' }; }; @@ -319,24 +319,24 @@ export const EnvSchema = z // Off by default: absence ⇒ disabled, so pure local deployments pay nothing. // When ENABLED=true the endpoint/bucket/creds are required together (a // partial config must refuse to boot) — enforced by the superRefine below. - OB_REPLICATION_ENABLED: envBoolean(false), + OPENBUCKET_REPLICATION_ENABLED: envBoolean(false), // S3-compatible endpoint (R2/B2/MinIO). Omit for real AWS S3 (the SDK derives // it from the region). http:// is accepted (warned at boot) for LAN dev. - OB_REPLICATION_ENDPOINT: z.string().optional(), - OB_REPLICATION_REGION: z.string().default('us-east-1'), - OB_REPLICATION_BUCKET: z.string().optional(), - OB_REPLICATION_ACCESS_KEY_ID: z.string().optional(), - OB_REPLICATION_SECRET_ACCESS_KEY: z.string().optional(), + OPENBUCKET_REPLICATION_ENDPOINT: z.string().optional(), + OPENBUCKET_REPLICATION_REGION: z.string().default('us-east-1'), + OPENBUCKET_REPLICATION_BUCKET: z.string().optional(), + OPENBUCKET_REPLICATION_ACCESS_KEY_ID: z.string().optional(), + OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY: z.string().optional(), // path-style addressing — true for MinIO / other S3-compat; false for AWS. - OB_REPLICATION_FORCE_PATH_STYLE: envBoolean(true), + OPENBUCKET_REPLICATION_FORCE_PATH_STYLE: envBoolean(true), // Dead-letter cap: after this many failed attempts an intent → `failed`. - OB_REPLICATION_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(50).default(12), + OPENBUCKET_REPLICATION_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(50).default(12), // Drain tick interval (ms). Floor 1000 so the drain can't hot-loop. - OB_REPLICATION_DRAIN_INTERVAL_MS: z.coerce.number().int().min(1_000).max(300_000).default(5_000), + OPENBUCKET_REPLICATION_DRAIN_INTERVAL_MS: z.coerce.number().int().min(1_000).max(300_000).default(5_000), // Distinct keys drained per tick — bounds per-tick work (CWE-770). - OB_REPLICATION_BATCH_KEYS: z.coerce.number().int().min(1).max(1_000).default(50), + OPENBUCKET_REPLICATION_BATCH_KEYS: z.coerce.number().int().min(1).max(1_000).default(50), // Objects larger than this stream via lib-storage multipart. Default 64 MiB. - OB_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES: z.coerce + OPENBUCKET_REPLICATION_LARGE_OBJECT_THRESHOLD_BYTES: z.coerce .number() .int() .positive() @@ -346,26 +346,26 @@ export const EnvSchema = z // Off by default: absence ⇒ disabled, so deployments that don't want an // automatic snapshot pay nothing. When ENABLED=true exactly one of // INTERVAL_MINUTES / CRON must be set (enforced by the superRefine below). - OB_SCHEDULED_BACKUP_ENABLED: envBoolean(false), + OPENBUCKET_SCHEDULED_BACKUP_ENABLED: envBoolean(false), // `instance` = one whole-instance snapshot; `buckets` = one snapshot per bucket. - OB_SCHEDULED_BACKUP_SCOPE: z.enum(['instance', 'buckets']).default('instance'), + OPENBUCKET_SCHEDULED_BACKUP_SCOPE: z.enum(['instance', 'buckets']).default('instance'), // Fixed interval between snapshots (minutes). Floor 5m so a misconfig can't // hammer the disk; ceiling 30d. Mutually exclusive with CRON. - OB_SCHEDULED_BACKUP_INTERVAL_MINUTES: z.coerce.number().int().min(5).max(43_200).optional(), + OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES: z.coerce.number().int().min(5).max(43_200).optional(), // 5-field cron schedule (validated by the superRefine below). Mutually // exclusive with INTERVAL_MINUTES. - OB_SCHEDULED_BACKUP_CRON: z.string().optional(), + OPENBUCKET_SCHEDULED_BACKUP_CRON: z.string().optional(), // Absolute snapshot directory. Defaults to `/backups` at resolve time. - OB_SCHEDULED_BACKUP_DIR: z.string().optional(), + OPENBUCKET_SCHEDULED_BACKUP_DIR: z.string().optional(), // Retention: keep the newest N snapshots (a hard floor — an old-but-within-N // snapshot is retained regardless of age). - OB_SCHEDULED_BACKUP_KEEP_LAST: z.coerce.number().int().min(1).max(1_000).default(7), + OPENBUCKET_SCHEDULED_BACKUP_KEEP_LAST: z.coerce.number().int().min(1).max(1_000).default(7), // Retention: also keep anything younger than this many days (union with // keep-last: a fresh snapshot is never deleted by the age rule). - OB_SCHEDULED_BACKUP_MAX_AGE_DAYS: z.coerce.number().int().min(1).max(3_650).default(30), + OPENBUCKET_SCHEDULED_BACKUP_MAX_AGE_DAYS: z.coerce.number().int().min(1).max(3_650).default(30), // Fixed wake tick: how often the runner checks whether a snapshot is due. // Floor 10s so a hostile-tiny value can't busy-loop the scheduler. - OB_SCHEDULED_BACKUP_CHECK_INTERVAL_MS: z.coerce + OPENBUCKET_SCHEDULED_BACKUP_CHECK_INTERVAL_MS: z.coerce .number() .int() .min(10_000) @@ -373,7 +373,7 @@ export const EnvSchema = z .default(60_000), // Also push each finished snapshot .zip to the replication target under a // reserved prefix. A no-op (with a boot warning) when replication is disabled. - OB_SCHEDULED_BACKUP_PUSH_TO_REPLICATION: envBoolean(false), + OPENBUCKET_SCHEDULED_BACKUP_PUSH_TO_REPLICATION: envBoolean(false), // --- cold-object tiering (STORY-0901) --- // Master switch; still a no-op unless a STORY-0900 remote target is configured. @@ -396,14 +396,14 @@ export const EnvSchema = z // The scrubber walks current/local objects, re-hashes each blob vs the stored // sha256, and marks a per-object verdict — strictly rate-limited so it never // starves request traffic. - OB_INTEGRITY_SCRUB_ENABLED: envBoolean(false), + OPENBUCKET_INTEGRITY_SCRUB_ENABLED: envBoolean(false), // Tick interval (ms). Floor 1s so a misconfig can't hot-loop the scheduler. - OB_INTEGRITY_SCRUB_INTERVAL_MS: z.coerce.number().int().min(1_000).default(60_000), + OPENBUCKET_INTEGRITY_SCRUB_INTERVAL_MS: z.coerce.number().int().min(1_000).default(60_000), // Hard per-tick object cap — bounds detection work regardless of blob sizes. - OB_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK: z.coerce.number().int().min(1).default(1_000), + OPENBUCKET_INTEGRITY_SCRUB_MAX_OBJECTS_PER_TICK: z.coerce.number().int().min(1).default(1_000), // Per-tick byte budget: stop the tick once this many bytes have been hashed // (disk-read amplification throttle). Default 1 GiB/tick. - OB_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK: z.coerce + OPENBUCKET_INTEGRITY_SCRUB_MAX_BYTES_PER_TICK: z.coerce .number() .int() .positive() @@ -455,30 +455,30 @@ export const EnvSchema = z // AWS but validated when present), bucket, and both credentials are required // together — a partial config must refuse to boot (mirrors the webhook / // admin-block footgun guards, fail-closed). - if (env.OB_REPLICATION_ENABLED) { + if (env.OPENBUCKET_REPLICATION_ENABLED) { const requireField = (key: keyof typeof env, label: string) => { if (!env[key]) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: [key as string], - message: `${label} is required when OB_REPLICATION_ENABLED=true`, + message: `${label} is required when OPENBUCKET_REPLICATION_ENABLED=true`, }); } }; - requireField('OB_REPLICATION_BUCKET', 'OB_REPLICATION_BUCKET'); - requireField('OB_REPLICATION_ACCESS_KEY_ID', 'OB_REPLICATION_ACCESS_KEY_ID'); - requireField('OB_REPLICATION_SECRET_ACCESS_KEY', 'OB_REPLICATION_SECRET_ACCESS_KEY'); - if (env.OB_REPLICATION_BUCKET && !S3_BUCKET_RE.test(env.OB_REPLICATION_BUCKET)) { + requireField('OPENBUCKET_REPLICATION_BUCKET', 'OPENBUCKET_REPLICATION_BUCKET'); + requireField('OPENBUCKET_REPLICATION_ACCESS_KEY_ID', 'OPENBUCKET_REPLICATION_ACCESS_KEY_ID'); + requireField('OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY', 'OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY'); + if (env.OPENBUCKET_REPLICATION_BUCKET && !S3_BUCKET_RE.test(env.OPENBUCKET_REPLICATION_BUCKET)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - path: ['OB_REPLICATION_BUCKET'], - message: 'OB_REPLICATION_BUCKET must be a valid S3 bucket name (3-63 chars)', + path: ['OPENBUCKET_REPLICATION_BUCKET'], + message: 'OPENBUCKET_REPLICATION_BUCKET must be a valid S3 bucket name (3-63 chars)', }); } - if (env.OB_REPLICATION_ENDPOINT) { - const { error } = validateReplicationEndpoint(env.OB_REPLICATION_ENDPOINT); + if (env.OPENBUCKET_REPLICATION_ENDPOINT) { + const { error } = validateReplicationEndpoint(env.OPENBUCKET_REPLICATION_ENDPOINT); if (error) { - ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['OB_REPLICATION_ENDPOINT'], message: error }); + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['OPENBUCKET_REPLICATION_ENDPOINT'], message: error }); } } } @@ -489,22 +489,22 @@ export const EnvSchema = z // push-to-replication with replication OFF is NOT a hard failure (the flag is // a no-op) — the runtime factory logs a boot WARNING so an operator can toggle // replication on later without being blocked here. - if (env.OB_SCHEDULED_BACKUP_ENABLED) { - const hasInterval = env.OB_SCHEDULED_BACKUP_INTERVAL_MINUTES != null; - const hasCron = env.OB_SCHEDULED_BACKUP_CRON != null && env.OB_SCHEDULED_BACKUP_CRON !== ''; + if (env.OPENBUCKET_SCHEDULED_BACKUP_ENABLED) { + const hasInterval = env.OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES != null; + const hasCron = env.OPENBUCKET_SCHEDULED_BACKUP_CRON != null && env.OPENBUCKET_SCHEDULED_BACKUP_CRON !== ''; if (hasInterval === hasCron) { ctx.addIssue({ code: z.ZodIssueCode.custom, - path: ['OB_SCHEDULED_BACKUP_CRON'], + path: ['OPENBUCKET_SCHEDULED_BACKUP_CRON'], message: - 'exactly one of OB_SCHEDULED_BACKUP_INTERVAL_MINUTES or OB_SCHEDULED_BACKUP_CRON ' + - 'must be set when OB_SCHEDULED_BACKUP_ENABLED=true', + 'exactly one of OPENBUCKET_SCHEDULED_BACKUP_INTERVAL_MINUTES or OPENBUCKET_SCHEDULED_BACKUP_CRON ' + + 'must be set when OPENBUCKET_SCHEDULED_BACKUP_ENABLED=true', }); } if (hasCron) { - const cronError = validateCronExpression(env.OB_SCHEDULED_BACKUP_CRON as string); + const cronError = validateCronExpression(env.OPENBUCKET_SCHEDULED_BACKUP_CRON as string); if (cronError) { - ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['OB_SCHEDULED_BACKUP_CRON'], message: cronError }); + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['OPENBUCKET_SCHEDULED_BACKUP_CRON'], message: cronError }); } } } diff --git a/libs/nestjs/src/lib/open-bucket-core.module.ts b/libs/nestjs/src/lib/open-bucket-core.module.ts index 3d48da6..96a96df 100644 --- a/libs/nestjs/src/lib/open-bucket-core.module.ts +++ b/libs/nestjs/src/lib/open-bucket-core.module.ts @@ -86,12 +86,12 @@ function buildCoreImports(adminEnabled: boolean): Array { // Replication target secret (STORY-0900). Same defence-in-depth: it // lives only in config / the S3Client credentials closure and is // never intentionally logged, but redact any `secretAccessKey` / - // `OB_REPLICATION_SECRET_ACCESS_KEY` field so a stray config or SDK + // `OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY` field so a stray config or SDK // request dump can't leak it, along with the SDK `authorization` header. 'secretAccessKey', '*.secretAccessKey', - 'OB_REPLICATION_SECRET_ACCESS_KEY', - '*.OB_REPLICATION_SECRET_ACCESS_KEY', + 'OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY', + '*.OPENBUCKET_REPLICATION_SECRET_ACCESS_KEY', // Scoped sub-key at-rest secret + KEK material (EPIC-11 / TASK-3001). // The blob is ciphertext and the KEK lives only in config, but redact // any `secretEncrypted` / `KEY_ENCRYPTION_SECRET` field so a stray diff --git a/libs/nestjs/src/lib/storage/replication/replication-config.ts b/libs/nestjs/src/lib/storage/replication/replication-config.ts index 76c0b11..c615eff 100644 --- a/libs/nestjs/src/lib/storage/replication/replication-config.ts +++ b/libs/nestjs/src/lib/storage/replication/replication-config.ts @@ -65,7 +65,7 @@ export function resolveReplicationConfig(config: AppConfigService): ReplicationC const { insecure } = validateReplicationEndpoint(endpoint); if (insecure) { new Logger('ReplicationConfig').warn( - `OB_REPLICATION_ENDPOINT uses plaintext http:// — replicated object bytes ` + + `OPENBUCKET_REPLICATION_ENDPOINT uses plaintext http:// — replicated object bytes ` + `(decrypted plaintext) will traverse the network unencrypted. Use https:// ` + `unless the target is on a trusted LAN (e.g. MinIO).`, );