Skip to content

Commit 755ee9e

Browse files
JSONboredJSONbored
committed
feat(selfhost): rotate subscription CLI credentials at runtime without a restart
Resolve the Claude OAuth token at AI-call time instead of freezing it into process.env at boot, and add three write surfaces for rotating it safely. Resolution order (highest first): a DB-backed fleet credential, a fresh read of CLAUDE_CODE_OAUTH_TOKEN_FILE, then the boot env value. The file is only re-read when the boot loader actually sourced the value from it, so the documented 'an inline .env value always wins' precedence is preserved; every failure degrades to the next rung rather than failing a review. Write surfaces: - scripts/rotate-secret.sh for a single box, validating shape and writing in place so the container's inode-pinned bind mount sees the change immediately - a rotate-secret verb on the redeploy companion, plus a loopover_admin_rotate_secret MCP admin tool, since the app container cannot write its own secrets (the mount is read-only) - INTERNAL_JOB_TOKEN-gated /v1/internal/provider-credentials/* backed by a new provider_credentials table, encrypted with the existing BYOK AES-256-GCM envelope, for fleets with no shared filesystem This closes two silent failure modes seen in production: a label line above the value became part of the credential (the loader only trims), and a write-new-then-rename left the running container serving the old value while still reporting healthy. Closes #9543
1 parent 8751b9a commit 755ee9e

25 files changed

Lines changed: 1789 additions & 15 deletions

‎apps/loopover-ui/public/openapi.json‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22690,6 +22690,217 @@
2269022690
}
2269122691
]
2269222692
}
22693+
},
22694+
"/v1/internal/provider-credentials/{provider}": {
22695+
"get": {
22696+
"summary": "Read the secret-free status of a stored instance subscription credential",
22697+
"parameters": [
22698+
{
22699+
"schema": {
22700+
"type": "string",
22701+
"enum": [
22702+
"claude-code",
22703+
"codex"
22704+
]
22705+
},
22706+
"required": true,
22707+
"name": "provider",
22708+
"in": "path"
22709+
}
22710+
],
22711+
"responses": {
22712+
"200": {
22713+
"description": "Credential status. Never includes the credential itself.",
22714+
"content": {
22715+
"application/json": {
22716+
"schema": {
22717+
"anyOf": [
22718+
{
22719+
"type": "object",
22720+
"properties": {
22721+
"configured": {
22722+
"type": "boolean",
22723+
"enum": [
22724+
false
22725+
]
22726+
}
22727+
},
22728+
"required": [
22729+
"configured"
22730+
]
22731+
},
22732+
{
22733+
"type": "object",
22734+
"properties": {
22735+
"configured": {
22736+
"type": "boolean",
22737+
"enum": [
22738+
true
22739+
]
22740+
},
22741+
"provider": {
22742+
"type": "string"
22743+
},
22744+
"last4": {
22745+
"type": "string"
22746+
},
22747+
"updatedBy": {
22748+
"type": "string",
22749+
"nullable": true
22750+
},
22751+
"updatedAt": {
22752+
"type": "string"
22753+
}
22754+
},
22755+
"required": [
22756+
"configured",
22757+
"provider",
22758+
"last4",
22759+
"updatedBy",
22760+
"updatedAt"
22761+
]
22762+
}
22763+
]
22764+
}
22765+
}
22766+
}
22767+
},
22768+
"400": {
22769+
"description": "Unknown provider"
22770+
},
22771+
"401": {
22772+
"description": "Invalid internal token"
22773+
}
22774+
},
22775+
"security": [
22776+
{
22777+
"LoopOverBearer": []
22778+
},
22779+
{
22780+
"LoopOverSessionCookie": []
22781+
}
22782+
]
22783+
},
22784+
"post": {
22785+
"summary": "Store or replace an instance subscription credential, encrypted at rest",
22786+
"parameters": [
22787+
{
22788+
"schema": {
22789+
"type": "string",
22790+
"enum": [
22791+
"claude-code",
22792+
"codex"
22793+
]
22794+
},
22795+
"required": true,
22796+
"name": "provider",
22797+
"in": "path"
22798+
}
22799+
],
22800+
"requestBody": {
22801+
"content": {
22802+
"application/json": {
22803+
"schema": {
22804+
"type": "object",
22805+
"properties": {
22806+
"credential": {
22807+
"type": "string"
22808+
}
22809+
},
22810+
"required": [
22811+
"credential"
22812+
]
22813+
}
22814+
}
22815+
}
22816+
},
22817+
"responses": {
22818+
"200": {
22819+
"description": "Credential stored. Returns the secret-free status.",
22820+
"content": {
22821+
"application/json": {
22822+
"schema": {
22823+
"type": "object",
22824+
"additionalProperties": {
22825+
"nullable": true
22826+
}
22827+
}
22828+
}
22829+
}
22830+
},
22831+
"400": {
22832+
"description": "Unknown provider, or a credential that is empty, padded, or not a single line"
22833+
},
22834+
"401": {
22835+
"description": "Invalid internal token"
22836+
},
22837+
"503": {
22838+
"description": "TOKEN_ENCRYPTION_SECRET is not configured, so the credential cannot be stored encrypted"
22839+
}
22840+
},
22841+
"security": [
22842+
{
22843+
"LoopOverBearer": []
22844+
},
22845+
{
22846+
"LoopOverSessionCookie": []
22847+
}
22848+
]
22849+
},
22850+
"delete": {
22851+
"summary": "Clear a stored instance subscription credential, falling back to the secret file or boot env",
22852+
"parameters": [
22853+
{
22854+
"schema": {
22855+
"type": "string",
22856+
"enum": [
22857+
"claude-code",
22858+
"codex"
22859+
]
22860+
},
22861+
"required": true,
22862+
"name": "provider",
22863+
"in": "path"
22864+
}
22865+
],
22866+
"responses": {
22867+
"200": {
22868+
"description": "Credential cleared",
22869+
"content": {
22870+
"application/json": {
22871+
"schema": {
22872+
"type": "object",
22873+
"properties": {
22874+
"configured": {
22875+
"type": "boolean",
22876+
"enum": [
22877+
false
22878+
]
22879+
}
22880+
},
22881+
"required": [
22882+
"configured"
22883+
]
22884+
}
22885+
}
22886+
}
22887+
},
22888+
"400": {
22889+
"description": "Unknown provider"
22890+
},
22891+
"401": {
22892+
"description": "Invalid internal token"
22893+
}
22894+
},
22895+
"security": [
22896+
{
22897+
"LoopOverBearer": []
22898+
},
22899+
{
22900+
"LoopOverSessionCookie": []
22901+
}
22902+
]
22903+
}
2269322904
}
2269422905
},
2269522906
"servers": [

‎migrations/0199_provider_credentials.sql‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Instance subscription-CLI credentials (#9543): the FLEET path for rotating CLAUDE_CODE_OAUTH_TOKEN (and
2+
-- the codex credential) without a restart. A single self-hosted box can rotate its credential in place on
3+
-- disk -- the secret file is a bind mount the running container re-reads at AI-call time -- but a
4+
-- multi-instance deployment has no shared filesystem, so the value lives here instead and every instance
5+
-- resolves it fresh per call (src/selfhost/provider-credential-registry.ts).
6+
--
7+
-- Keyed by PROVIDER, not by repo: unlike repository_ai_keys / repository_linear_keys (the per-maintainer
8+
-- BYOK tables this deliberately mirrors), this is the instance's OWN subscription credential, so there is
9+
-- exactly one row per provider. Encrypted at rest with the same AES-256-GCM envelope and the same
10+
-- TOKEN_ENCRYPTION_SECRET (see src/utils/crypto.ts); `last4` is a display-only hint derived from the
11+
-- plaintext at write time, and the plaintext is never stored, never logged, and never returned by the API.
12+
--
13+
-- No DB-side DEFAULT CURRENT_TIMESTAMP on created_at/updated_at, matching migrations/0111_linear_backend.sql:
14+
-- every write goes through Drizzle's $defaultFn(() => nowIso()) (src/db/schema.ts), which always supplies the
15+
-- ISO timestamp explicitly, so a SQLite-format fallback here would be unused surface area, not a safeguard.
16+
CREATE TABLE IF NOT EXISTS provider_credentials (
17+
provider TEXT PRIMARY KEY,
18+
ciphertext TEXT NOT NULL,
19+
iv TEXT NOT NULL,
20+
salt TEXT,
21+
key_version INTEGER NOT NULL DEFAULT 1,
22+
last4 TEXT NOT NULL,
23+
updated_by TEXT,
24+
created_at TEXT NOT NULL,
25+
updated_at TEXT NOT NULL
26+
);

‎scripts/redeploy-companion.ts‎

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import { createServer } from "node:net";
3232
import { spawn } from "node:child_process";
3333
import { timingSafeEqual } from "node:crypto";
34-
import { unlinkSync, chmodSync, existsSync } from "node:fs";
34+
import { unlinkSync, chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
3535

3636
const SOCKET_PATH = process.env.REDEPLOY_COMPANION_SOCKET_PATH?.trim() || "/run/loopover-redeploy.sock";
3737
const REPO_ROOT = process.env.REDEPLOY_COMPANION_REPO_ROOT?.trim() || process.cwd();
@@ -56,7 +56,9 @@ function isValidToken(configuredToken: string, candidate: unknown): boolean {
5656
return timingSafeEqual(configured, supplied);
5757
}
5858

59-
type RedeployRequest = { token: unknown; image?: unknown };
59+
// `action` is absent on every pre-#9543 request, and MUST keep meaning "redeploy" -- the app container and
60+
// the host companion are upgraded independently, so a new companion always has to serve an old client.
61+
type RedeployRequest = { token: unknown; image?: unknown; action?: unknown; secret?: unknown; value?: unknown };
6062

6163
function parseRequestLine(line: string): RedeployRequest | null {
6264
let parsed: unknown;
@@ -83,6 +85,82 @@ function isSafeImageOverride(value: unknown): value is string {
8385
return typeof value === "string" && value.length > 0 && value.length <= 512 && !/[\s"'\\${}`;|&<>]/.test(value);
8486
}
8587

88+
// ─── Secret rotation (#9543) ────────────────────────────────────────────────────────────────────
89+
// The SECOND verb this companion serves. It lives here, host-side, for a hard reason: the app container
90+
// physically cannot write its own credential -- docker inspect reports the Compose `secrets:` bind mount
91+
// as rw=false -- so a container-side rotation endpoint is impossible, not merely undesirable.
92+
//
93+
// Two footguns this exists to make unhittable, both of which fail SILENTLY (the container stays healthy
94+
// and the status stays green while reviews degrade to the fallback provider):
95+
// 1. Shape: src/selfhost/load-file-secrets.ts only .trim()s the file. A human-added label line above the
96+
// value becomes part of the credential. Hence the single-line/no-comment/no-whitespace validation.
97+
// 2. Inode: a Compose secret is a plain bind mount pinned to the INODE. Writing in place propagates to
98+
// the running container instantly; write-new-then-rename (`mv`, and several editors' default save)
99+
// leaves the container serving the OLD content, and `docker compose up -d` will NOT fix it -- it
100+
// reports "Container Running" and changes nothing, because the Compose config is unchanged. So this
101+
// writes with truncate-in-place and never renames.
102+
103+
/** Secrets this verb may write, mapped to their path relative to REPO_ROOT. An explicit allowlist rather
104+
* than an arbitrary operator-supplied path: the whole point is that this process runs with more host
105+
* privilege than its caller, so it must never become a general "write any file as me" primitive. */
106+
const ROTATABLE_SECRETS: Record<string, string> = {
107+
claude_code_oauth_token: "secrets/claude_code_oauth_token.txt",
108+
github_webhook_secret: "secrets/github_webhook_secret.txt",
109+
loopover_api_token: "secrets/loopover_api_token.txt",
110+
loopover_mcp_token: "secrets/loopover_mcp_token.txt",
111+
loopover_mcp_admin_token: "secrets/loopover_mcp_admin_token.txt",
112+
pagerduty_routing_key: "secrets/pagerduty_routing_key.txt",
113+
};
114+
115+
/** A secret file's value must be exactly the credential -- one line, no comment, no surrounding
116+
* whitespace, non-empty, and bounded. Rejecting here is what turns footgun 1 above into an error the
117+
* operator sees immediately instead of a silent provider downgrade discovered days later. */
118+
export function isValidSecretValue(value: unknown): value is string {
119+
return typeof value === "string" && value.length > 0 && value.length <= 4096 && !/[\r\n]/.test(value) && value.trim() === value && !value.startsWith("#");
120+
}
121+
122+
export type RotateSecretResult = { ok: boolean; error?: string; backupPath?: string };
123+
124+
/**
125+
* Write a secret file IN PLACE (truncate, never rename) so the running container's bind mount -- pinned to
126+
* the inode -- sees the new bytes immediately. Backs the previous value up first, and preserves the 0644
127+
* the app's own uid depends on to read it back (secrets/README.md explains why 600 breaks the app).
128+
*
129+
* Injectable fs/now for tests only; production always uses the real node:fs.
130+
*/
131+
export function rotateSecret(
132+
name: unknown,
133+
value: unknown,
134+
io: {
135+
readFileSync: typeof readFileSync;
136+
writeFileSync: typeof writeFileSync;
137+
existsSync: typeof existsSync;
138+
chmodSync: typeof chmodSync;
139+
now: () => Date;
140+
} = { readFileSync, writeFileSync, existsSync, chmodSync, now: () => new Date() },
141+
): RotateSecretResult {
142+
if (typeof name !== "string" || !Object.hasOwn(ROTATABLE_SECRETS, name)) return { ok: false, error: "unknown_secret" };
143+
if (!isValidSecretValue(value)) return { ok: false, error: "invalid_secret_value" };
144+
const target = `${REPO_ROOT}/${ROTATABLE_SECRETS[name]}`;
145+
try {
146+
let backupPath: string | undefined;
147+
if (io.existsSync(target)) {
148+
const previous = io.readFileSync(target, "utf8");
149+
// Backups live beside the deploy backups, never inside secrets/ -- that directory is the Compose
150+
// `secrets:` source, and a stray file there is one careless glob away from being mounted somewhere.
151+
backupPath = `${REPO_ROOT}/.deploy-backups/${name}.txt.bak-${io.now().toISOString().replace(/[:.]/g, "").replace(/-/g, "")}`;
152+
io.writeFileSync(backupPath, previous, { mode: 0o600 });
153+
}
154+
// No trailing newline: secrets/README.md's own `printf '%s'` convention. The loader .trim()s anyway,
155+
// so this is about keeping the file byte-identical to the issued credential, not correctness.
156+
io.writeFileSync(target, value, { mode: 0o644, flag: "w" });
157+
io.chmodSync(target, 0o644); // an existing file keeps its old mode through a plain write -- force it
158+
return backupPath ? { ok: true, backupPath } : { ok: true };
159+
} catch (error) {
160+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
161+
}
162+
}
163+
86164
export type RunDeployResult = { ok: boolean; exitCode: number | null; error?: string };
87165

88166
/** Runs the real, existing deploy-selfhost-image.sh -- never reimplemented here. Injectable spawn function for
@@ -116,12 +194,24 @@ export async function handleConnection(
116194
setBusy: (busy: boolean) => void,
117195
write: (line: string) => void,
118196
deploy: typeof runDeploy = runDeploy,
197+
rotate: typeof rotateSecret = rotateSecret,
119198
): Promise<void> {
120199
const request = parseRequestLine(requestLine);
121200
if (!request || !isValidToken(configuredToken, request.token)) {
122201
write(JSON.stringify({ ok: false, error: "unauthorized" }));
123202
return;
124203
}
204+
// Rotation is answered BEFORE the busy check: it is a single truncating write, not orchestration, so it
205+
// neither races a running redeploy nor is worth refusing during one -- and refusing it during a 15-minute
206+
// redeploy is exactly when an operator is most likely to need a credential fixed.
207+
if (request.action === "rotate-secret") {
208+
write(JSON.stringify(rotate(request.secret, request.value)));
209+
return;
210+
}
211+
if (request.action !== undefined && request.action !== "redeploy") {
212+
write(JSON.stringify({ ok: false, error: "unknown_action" }));
213+
return;
214+
}
125215
if (isBusy()) {
126216
write(JSON.stringify({ ok: false, error: "redeploy_already_in_progress" }));
127217
return;

0 commit comments

Comments
 (0)