Skip to content

Commit c2d6102

Browse files
JSONboredJSONbored
authored
feat(selfhost): rotate subscription CLI credentials at runtime without a restart (#9548)
Resolve the Claude OAuth token at AI-call time instead of freezing it into process.env at boot, and add three write surfaces for rotating it safely. Resolution order (highest first): a DB-backed fleet credential, a fresh read of CLAUDE_CODE_OAUTH_TOKEN_FILE, then the boot env value. The file is only re-read when the boot loader actually sourced the value from it, so the documented 'an inline .env value always wins' precedence is preserved; every failure degrades to the next rung rather than failing a review. Write surfaces: - scripts/rotate-secret.sh for a single box, validating shape and writing in place so the container's inode-pinned bind mount sees the change immediately - a rotate-secret verb on the redeploy companion, plus a loopover_admin_rotate_secret MCP admin tool, since the app container cannot write its own secrets (the mount is read-only) - INTERNAL_JOB_TOKEN-gated /v1/internal/provider-credentials/* backed by a new provider_credentials table, encrypted with the existing BYOK AES-256-GCM envelope, for fleets with no shared filesystem This closes two silent failure modes seen in production: a label line above the value became part of the credential (the loader only trims), and a write-new-then-rename left the running container serving the old value while still reporting healthy. Closes #9543
1 parent 78061a0 commit c2d6102

25 files changed

Lines changed: 1789 additions & 15 deletions

‎apps/loopover-ui/public/openapi.json‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22755,6 +22755,217 @@
2275522755
}
2275622756
]
2275722757
}
22758+
},
22759+
"/v1/internal/provider-credentials/{provider}": {
22760+
"get": {
22761+
"summary": "Read the secret-free status of a stored instance subscription credential",
22762+
"parameters": [
22763+
{
22764+
"schema": {
22765+
"type": "string",
22766+
"enum": [
22767+
"claude-code",
22768+
"codex"
22769+
]
22770+
},
22771+
"required": true,
22772+
"name": "provider",
22773+
"in": "path"
22774+
}
22775+
],
22776+
"responses": {
22777+
"200": {
22778+
"description": "Credential status. Never includes the credential itself.",
22779+
"content": {
22780+
"application/json": {
22781+
"schema": {
22782+
"anyOf": [
22783+
{
22784+
"type": "object",
22785+
"properties": {
22786+
"configured": {
22787+
"type": "boolean",
22788+
"enum": [
22789+
false
22790+
]
22791+
}
22792+
},
22793+
"required": [
22794+
"configured"
22795+
]
22796+
},
22797+
{
22798+
"type": "object",
22799+
"properties": {
22800+
"configured": {
22801+
"type": "boolean",
22802+
"enum": [
22803+
true
22804+
]
22805+
},
22806+
"provider": {
22807+
"type": "string"
22808+
},
22809+
"last4": {
22810+
"type": "string"
22811+
},
22812+
"updatedBy": {
22813+
"type": "string",
22814+
"nullable": true
22815+
},
22816+
"updatedAt": {
22817+
"type": "string"
22818+
}
22819+
},
22820+
"required": [
22821+
"configured",
22822+
"provider",
22823+
"last4",
22824+
"updatedBy",
22825+
"updatedAt"
22826+
]
22827+
}
22828+
]
22829+
}
22830+
}
22831+
}
22832+
},
22833+
"400": {
22834+
"description": "Unknown provider"
22835+
},
22836+
"401": {
22837+
"description": "Invalid internal token"
22838+
}
22839+
},
22840+
"security": [
22841+
{
22842+
"LoopOverBearer": []
22843+
},
22844+
{
22845+
"LoopOverSessionCookie": []
22846+
}
22847+
]
22848+
},
22849+
"post": {
22850+
"summary": "Store or replace an instance subscription credential, encrypted at rest",
22851+
"parameters": [
22852+
{
22853+
"schema": {
22854+
"type": "string",
22855+
"enum": [
22856+
"claude-code",
22857+
"codex"
22858+
]
22859+
},
22860+
"required": true,
22861+
"name": "provider",
22862+
"in": "path"
22863+
}
22864+
],
22865+
"requestBody": {
22866+
"content": {
22867+
"application/json": {
22868+
"schema": {
22869+
"type": "object",
22870+
"properties": {
22871+
"credential": {
22872+
"type": "string"
22873+
}
22874+
},
22875+
"required": [
22876+
"credential"
22877+
]
22878+
}
22879+
}
22880+
}
22881+
},
22882+
"responses": {
22883+
"200": {
22884+
"description": "Credential stored. Returns the secret-free status.",
22885+
"content": {
22886+
"application/json": {
22887+
"schema": {
22888+
"type": "object",
22889+
"additionalProperties": {
22890+
"nullable": true
22891+
}
22892+
}
22893+
}
22894+
}
22895+
},
22896+
"400": {
22897+
"description": "Unknown provider, or a credential that is empty, padded, or not a single line"
22898+
},
22899+
"401": {
22900+
"description": "Invalid internal token"
22901+
},
22902+
"503": {
22903+
"description": "TOKEN_ENCRYPTION_SECRET is not configured, so the credential cannot be stored encrypted"
22904+
}
22905+
},
22906+
"security": [
22907+
{
22908+
"LoopOverBearer": []
22909+
},
22910+
{
22911+
"LoopOverSessionCookie": []
22912+
}
22913+
]
22914+
},
22915+
"delete": {
22916+
"summary": "Clear a stored instance subscription credential, falling back to the secret file or boot env",
22917+
"parameters": [
22918+
{
22919+
"schema": {
22920+
"type": "string",
22921+
"enum": [
22922+
"claude-code",
22923+
"codex"
22924+
]
22925+
},
22926+
"required": true,
22927+
"name": "provider",
22928+
"in": "path"
22929+
}
22930+
],
22931+
"responses": {
22932+
"200": {
22933+
"description": "Credential cleared",
22934+
"content": {
22935+
"application/json": {
22936+
"schema": {
22937+
"type": "object",
22938+
"properties": {
22939+
"configured": {
22940+
"type": "boolean",
22941+
"enum": [
22942+
false
22943+
]
22944+
}
22945+
},
22946+
"required": [
22947+
"configured"
22948+
]
22949+
}
22950+
}
22951+
}
22952+
},
22953+
"400": {
22954+
"description": "Unknown provider"
22955+
},
22956+
"401": {
22957+
"description": "Invalid internal token"
22958+
}
22959+
},
22960+
"security": [
22961+
{
22962+
"LoopOverBearer": []
22963+
},
22964+
{
22965+
"LoopOverSessionCookie": []
22966+
}
22967+
]
22968+
}
2275822969
}
2275922970
},
2276022971
"servers": [

‎migrations/0199_provider_credentials.sql‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Instance subscription-CLI credentials (#9543): the FLEET path for rotating CLAUDE_CODE_OAUTH_TOKEN (and
2+
-- the codex credential) without a restart. A single self-hosted box can rotate its credential in place on
3+
-- disk -- the secret file is a bind mount the running container re-reads at AI-call time -- but a
4+
-- multi-instance deployment has no shared filesystem, so the value lives here instead and every instance
5+
-- resolves it fresh per call (src/selfhost/provider-credential-registry.ts).
6+
--
7+
-- Keyed by PROVIDER, not by repo: unlike repository_ai_keys / repository_linear_keys (the per-maintainer
8+
-- BYOK tables this deliberately mirrors), this is the instance's OWN subscription credential, so there is
9+
-- exactly one row per provider. Encrypted at rest with the same AES-256-GCM envelope and the same
10+
-- TOKEN_ENCRYPTION_SECRET (see src/utils/crypto.ts); `last4` is a display-only hint derived from the
11+
-- plaintext at write time, and the plaintext is never stored, never logged, and never returned by the API.
12+
--
13+
-- No DB-side DEFAULT CURRENT_TIMESTAMP on created_at/updated_at, matching migrations/0111_linear_backend.sql:
14+
-- every write goes through Drizzle's $defaultFn(() => nowIso()) (src/db/schema.ts), which always supplies the
15+
-- ISO timestamp explicitly, so a SQLite-format fallback here would be unused surface area, not a safeguard.
16+
CREATE TABLE IF NOT EXISTS provider_credentials (
17+
provider TEXT PRIMARY KEY,
18+
ciphertext TEXT NOT NULL,
19+
iv TEXT NOT NULL,
20+
salt TEXT,
21+
key_version INTEGER NOT NULL DEFAULT 1,
22+
last4 TEXT NOT NULL,
23+
updated_by TEXT,
24+
created_at TEXT NOT NULL,
25+
updated_at TEXT NOT NULL
26+
);

‎scripts/redeploy-companion.ts‎

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import { createServer } from "node:net";
3232
import { spawn } from "node:child_process";
3333
import { timingSafeEqual } from "node:crypto";
34-
import { unlinkSync, chmodSync, existsSync } from "node:fs";
34+
import { unlinkSync, chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
3535

3636
const SOCKET_PATH = process.env.REDEPLOY_COMPANION_SOCKET_PATH?.trim() || "/run/loopover-redeploy.sock";
3737
const REPO_ROOT = process.env.REDEPLOY_COMPANION_REPO_ROOT?.trim() || process.cwd();
@@ -56,7 +56,9 @@ function isValidToken(configuredToken: string, candidate: unknown): boolean {
5656
return timingSafeEqual(configured, supplied);
5757
}
5858

59-
type RedeployRequest = { token: unknown; image?: unknown };
59+
// `action` is absent on every pre-#9543 request, and MUST keep meaning "redeploy" -- the app container and
60+
// the host companion are upgraded independently, so a new companion always has to serve an old client.
61+
type RedeployRequest = { token: unknown; image?: unknown; action?: unknown; secret?: unknown; value?: unknown };
6062

6163
function parseRequestLine(line: string): RedeployRequest | null {
6264
let parsed: unknown;
@@ -83,6 +85,82 @@ function isSafeImageOverride(value: unknown): value is string {
8385
return typeof value === "string" && value.length > 0 && value.length <= 512 && !/[\s"'\\${}`;|&<>]/.test(value);
8486
}
8587

88+
// ─── Secret rotation (#9543) ────────────────────────────────────────────────────────────────────
89+
// The SECOND verb this companion serves. It lives here, host-side, for a hard reason: the app container
90+
// physically cannot write its own credential -- docker inspect reports the Compose `secrets:` bind mount
91+
// as rw=false -- so a container-side rotation endpoint is impossible, not merely undesirable.
92+
//
93+
// Two footguns this exists to make unhittable, both of which fail SILENTLY (the container stays healthy
94+
// and the status stays green while reviews degrade to the fallback provider):
95+
// 1. Shape: src/selfhost/load-file-secrets.ts only .trim()s the file. A human-added label line above the
96+
// value becomes part of the credential. Hence the single-line/no-comment/no-whitespace validation.
97+
// 2. Inode: a Compose secret is a plain bind mount pinned to the INODE. Writing in place propagates to
98+
// the running container instantly; write-new-then-rename (`mv`, and several editors' default save)
99+
// leaves the container serving the OLD content, and `docker compose up -d` will NOT fix it -- it
100+
// reports "Container Running" and changes nothing, because the Compose config is unchanged. So this
101+
// writes with truncate-in-place and never renames.
102+
103+
/** Secrets this verb may write, mapped to their path relative to REPO_ROOT. An explicit allowlist rather
104+
* than an arbitrary operator-supplied path: the whole point is that this process runs with more host
105+
* privilege than its caller, so it must never become a general "write any file as me" primitive. */
106+
const ROTATABLE_SECRETS: Record<string, string> = {
107+
claude_code_oauth_token: "secrets/claude_code_oauth_token.txt",
108+
github_webhook_secret: "secrets/github_webhook_secret.txt",
109+
loopover_api_token: "secrets/loopover_api_token.txt",
110+
loopover_mcp_token: "secrets/loopover_mcp_token.txt",
111+
loopover_mcp_admin_token: "secrets/loopover_mcp_admin_token.txt",
112+
pagerduty_routing_key: "secrets/pagerduty_routing_key.txt",
113+
};
114+
115+
/** A secret file's value must be exactly the credential -- one line, no comment, no surrounding
116+
* whitespace, non-empty, and bounded. Rejecting here is what turns footgun 1 above into an error the
117+
* operator sees immediately instead of a silent provider downgrade discovered days later. */
118+
export function isValidSecretValue(value: unknown): value is string {
119+
return typeof value === "string" && value.length > 0 && value.length <= 4096 && !/[\r\n]/.test(value) && value.trim() === value && !value.startsWith("#");
120+
}
121+
122+
export type RotateSecretResult = { ok: boolean; error?: string; backupPath?: string };
123+
124+
/**
125+
* Write a secret file IN PLACE (truncate, never rename) so the running container's bind mount -- pinned to
126+
* the inode -- sees the new bytes immediately. Backs the previous value up first, and preserves the 0644
127+
* the app's own uid depends on to read it back (secrets/README.md explains why 600 breaks the app).
128+
*
129+
* Injectable fs/now for tests only; production always uses the real node:fs.
130+
*/
131+
export function rotateSecret(
132+
name: unknown,
133+
value: unknown,
134+
io: {
135+
readFileSync: typeof readFileSync;
136+
writeFileSync: typeof writeFileSync;
137+
existsSync: typeof existsSync;
138+
chmodSync: typeof chmodSync;
139+
now: () => Date;
140+
} = { readFileSync, writeFileSync, existsSync, chmodSync, now: () => new Date() },
141+
): RotateSecretResult {
142+
if (typeof name !== "string" || !Object.hasOwn(ROTATABLE_SECRETS, name)) return { ok: false, error: "unknown_secret" };
143+
if (!isValidSecretValue(value)) return { ok: false, error: "invalid_secret_value" };
144+
const target = `${REPO_ROOT}/${ROTATABLE_SECRETS[name]}`;
145+
try {
146+
let backupPath: string | undefined;
147+
if (io.existsSync(target)) {
148+
const previous = io.readFileSync(target, "utf8");
149+
// Backups live beside the deploy backups, never inside secrets/ -- that directory is the Compose
150+
// `secrets:` source, and a stray file there is one careless glob away from being mounted somewhere.
151+
backupPath = `${REPO_ROOT}/.deploy-backups/${name}.txt.bak-${io.now().toISOString().replace(/[:.]/g, "").replace(/-/g, "")}`;
152+
io.writeFileSync(backupPath, previous, { mode: 0o600 });
153+
}
154+
// No trailing newline: secrets/README.md's own `printf '%s'` convention. The loader .trim()s anyway,
155+
// so this is about keeping the file byte-identical to the issued credential, not correctness.
156+
io.writeFileSync(target, value, { mode: 0o644, flag: "w" });
157+
io.chmodSync(target, 0o644); // an existing file keeps its old mode through a plain write -- force it
158+
return backupPath ? { ok: true, backupPath } : { ok: true };
159+
} catch (error) {
160+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
161+
}
162+
}
163+
86164
export type RunDeployResult = { ok: boolean; exitCode: number | null; error?: string };
87165

88166
/** Runs the real, existing deploy-selfhost-image.sh -- never reimplemented here. Injectable spawn function for
@@ -116,12 +194,24 @@ export async function handleConnection(
116194
setBusy: (busy: boolean) => void,
117195
write: (line: string) => void,
118196
deploy: typeof runDeploy = runDeploy,
197+
rotate: typeof rotateSecret = rotateSecret,
119198
): Promise<void> {
120199
const request = parseRequestLine(requestLine);
121200
if (!request || !isValidToken(configuredToken, request.token)) {
122201
write(JSON.stringify({ ok: false, error: "unauthorized" }));
123202
return;
124203
}
204+
// Rotation is answered BEFORE the busy check: it is a single truncating write, not orchestration, so it
205+
// neither races a running redeploy nor is worth refusing during one -- and refusing it during a 15-minute
206+
// redeploy is exactly when an operator is most likely to need a credential fixed.
207+
if (request.action === "rotate-secret") {
208+
write(JSON.stringify(rotate(request.secret, request.value)));
209+
return;
210+
}
211+
if (request.action !== undefined && request.action !== "redeploy") {
212+
write(JSON.stringify({ ok: false, error: "unknown_action" }));
213+
return;
214+
}
125215
if (isBusy()) {
126216
write(JSON.stringify({ ok: false, error: "redeploy_already_in_progress" }));
127217
return;

0 commit comments

Comments
 (0)