diff --git a/AGENTS.md b/AGENTS.md index 9f7b5a89..1489dedf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Documentation -User-, operator-, and admin-facing documentation — installing a watcher, adding an instrument, managing tokens, security/permissions — lives on the docs site at https://datahub.arcadiascience.com/docs, **not in this repository**. Search there first for "how do I use Data Hub" questions; don't rely on training data or guess at UI flows, since the site's `/docs/llms.txt` and `/docs/llms-full.txt` routes (and a `.md` suffix on any page URL) serve clean Markdown that's cheap to fetch. +User-, operator-, and admin-facing documentation — installing a watcher, setting up an instrument, managing tokens, security/permissions — lives on the docs site at https://datahub.arcadiascience.com/docs, **not in this repository**. Search there first for "how do I use Data Hub" questions; don't rely on training data or guess at UI flows, since the site's `/docs/llms.txt` and `/docs/llms-full.txt` routes (and a `.md` suffix on any page URL) serve clean Markdown that's cheap to fetch. This repo's `developer-docs/` covers contributing to and self-hosting Data Hub itself: architecture internals, local dev setup (`getting-started.md`, `local-development.md`), conventions, the step-by-step self-hosting guide for the web app and AWS infrastructure (`first-time-deployment.md`) plus CI/ongoing-deploy reference (`ci-and-deployment.md`), and per-package references (`lambda.md`, `watcher.md`, `shared-library.md`). See `developer-docs/README.md` for the full index. diff --git a/README.md b/README.md index ab977400..b6ad27e7 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ make dev See the full [Getting started guide](developer-docs/getting-started.md) for prerequisites and details. Don't have AWS/Google credentials? [Local development](developer-docs/local-development.md) covers a zero-credential setup for the web app + API + database alone (no watcher or Lambda needed). -Developer docs live in [developer-docs/](developer-docs/README.md), including the step-by-step guide to self-hosting Data Hub (deploying the web app and AWS infrastructure — see [First-time deployment](developer-docs/first-time-deployment.md)). You can find user documentation (watcher installation, adding an instrument, managing tokens) on the [docs site](https://datahub.arcadiascience.com/docs). +Developer docs live in [developer-docs/](developer-docs/README.md), including the step-by-step guide to self-hosting Data Hub (deploying the web app and AWS infrastructure — see [First-time deployment](developer-docs/first-time-deployment.md)). You can find user documentation (watcher installation, setting up an instrument, managing tokens) on the [docs site](https://datahub.arcadiascience.com/docs). ## Checks and tests diff --git a/developer-docs/architecture.md b/developer-docs/architecture.md index dc34060e..cb0c5a96 100644 --- a/developer-docs/architecture.md +++ b/developer-docs/architecture.md @@ -58,6 +58,6 @@ Steps 1–3 are the same, but the watcher does not upload immediately. Instead: - **Lambda-built run archives.** The "Download all" actions on a run delegate to the Lambda, which streams files from the raw + processed buckets directly into a zip in a separate `arcadia-data-hub-archives-{env}` bucket via S3 multipart upload. The web app then 302s the browser to a short-lived presigned URL on that archive — bytes never traverse Vercel, eliminating Fast Origin Transfer for run downloads. Builds are cached at `runs/{instrument_id}/{run_id}/{fingerprint}.zip` and expire after 7 days. Every cache miss is dispatched asynchronously: the route inserts an `archive_jobs` row, schedules the Lambda invocation via `after()`, returns `202 { job_id }`, and the UI polls the same `/download-archive` URL (which HEADs S3 on each call) until the artifact appears — making the S3 object, not the row's `status`, the source of truth for "ready". See [Run archives](run-archives.md) for the full flow and runbook. - **Public page metadata, gated bodies.** Routes commonly shared into Slack/Notion (dashboard, instruments, instrument and run detail, settings) are reachable without a session so link unfurlers can read `` metadata; the page or layout renders a `SignInRequired` CTA in place of the real body when there's no session. `/watchers/*` stays redirected to `/login` by `web/proxy.ts`. Three independent layers prevent search indexing: a `robots` field on the root layout, an `app/robots.ts` that disallows all generic crawlers (with an allow-list for unfurl bots), and an `X-Robots-Tag: noindex, nofollow` header on every response. - **Shared library for contracts.** Instrument IDs, S3 utilities, and environment config live in `data-hub-shared` so they stay consistent across Lambda and the watcher without duplicating code. -- **Wildcard PAT scope for the legacy backfill.** Migration `0022_pat_scopes` backfilled every pre-existing token with `["*"]` so deployed watchers and the Lambda kept working once scope enforcement shipped. `POST /api/v1/tokens` rejects `*` from API callers, so every token minted since carries explicit least-privilege scopes; see [Security & permissions](https://datahub.arcadiascience.com/docs/security#token-scopes) for the scope vocabulary. -- **MCP for AI access.** The web app includes a [Model Context Protocol](https://modelcontextprotocol.io/) server at `/api/v1/mcp` that exposes tools (read + scoped writes), resources, and prompts. AI clients (e.g. Claude Desktop, Cursor) authenticate with a personal access token and can browse instruments/runs, download files, claim runs, comment, reprocess, and inspect watcher health. See the [MCP server docs](https://datahub.arcadiascience.com/docs/mcp-server). +- **Wildcard PAT scope for the legacy backfill.** Migration `0022_pat_scopes` backfilled every pre-existing token with `["*"]` so deployed watchers and the Lambda kept working once scope enforcement shipped. `POST /api/v1/tokens` rejects `*` from API callers, so every token minted since carries explicit least-privilege scopes; see [Security and permissions](https://datahub.arcadiascience.com/docs/security#token-scopes) for the scope vocabulary. +- **MCP for AI access.** The web app includes a [Model Context Protocol](https://modelcontextprotocol.io/) server at `/api/v1/mcp` that exposes tools (read + scoped writes), resources, and prompts. AI clients (e.g. Claude Desktop, Cursor) authenticate with a personal access token and can browse instruments/runs, download files, claim runs, comment, reprocess, and inspect watcher health. See the [MCP overview](https://datahub.arcadiascience.com/docs/mcp). - **Integration tests against a real server.** The shared `testing.py` module spins up a real Next.js server backed by a Postgres database, so Lambda and watcher integration tests exercise the actual API surface. diff --git a/developer-docs/ci-and-deployment.md b/developer-docs/ci-and-deployment.md index 5def2ba1..e62e02a2 100644 --- a/developer-docs/ci-and-deployment.md +++ b/developer-docs/ci-and-deployment.md @@ -164,7 +164,7 @@ The CI deploy role has permission to roll new triggers out, so the trigger goes ### Watcher (PyPI) -The `data-hub-watcher` Python package is published to [PyPI](https://pypi.org/project/data-hub-watcher/) so lab PCs can install and self-update via `uv tool install data-hub-watcher`. The full release flow — version bump, tag, approval, env-var roll-out, mandatory updates, and rollback — is documented in the admin-facing [Managing watchers → Releases and fleet updates](https://datahub.arcadiascience.com/docs/managing-watchers#releases-and-fleet-updates) guide; this section is intentionally a pointer rather than a second source of truth so the two can't drift. +The `data-hub-watcher` Python package is published to [PyPI](https://pypi.org/project/data-hub-watcher/) so lab PCs can install and self-update via `uv tool install data-hub-watcher`. The full release flow — version bump, tag, approval, env-var roll-out, mandatory updates, and rollback — is documented in the admin-facing [Roll out watcher releases](https://datahub.arcadiascience.com/docs/watcher-releases) guide; this section is intentionally a pointer rather than a second source of truth so the two can't drift. Trusted publishing is configured under **Project → Publishing** on PyPI for `Arcadia-Science/data-hub` and the workflow `publish-watcher.yml`; no API token lives in repo secrets. If trust is ever revoked or rotated, update it there and re-run the workflow. diff --git a/developer-docs/first-time-deployment.md b/developer-docs/first-time-deployment.md index bda939e4..54a23391 100644 --- a/developer-docs/first-time-deployment.md +++ b/developer-docs/first-time-deployment.md @@ -82,7 +82,7 @@ npm run db:migrate ### Create an API key for the Lambda -Sign in with an account listed in `ADMIN_EMAILS`, then create a personal access token under Settings. The AWS stack and the Lambda use this token as `DATA_HUB_API_KEY` to call the Data Hub API, so create it now and keep it for [step 4](#4-deploy-the-aws-infrastructure). See [Managing tokens](https://datahub.arcadiascience.com/docs/managing-tokens) for the token UI. +Sign in with an account listed in `ADMIN_EMAILS`, then create a personal access token under Settings. The AWS stack and the Lambda use this token as `DATA_HUB_API_KEY` to call the Data Hub API, so create it now and keep it for [step 4](#4-deploy-the-aws-infrastructure). See [Issue and revoke tokens](https://datahub.arcadiascience.com/docs/manage-tokens) for the token UI. ## 3. Bootstrap AWS (once per account) @@ -180,4 +180,4 @@ The S3 bucket names default to `arcadia-data-hub-raw-` and `arcadia-data-hu ## After the backend is up -The environment is ready: lab operators can install watchers and start uploading. Point them at the [Quickstart](https://datahub.arcadiascience.com/docs/quickstart). Every subsequent deploy — web app, migrations, and Lambda — runs through CI; see [CI and deployment](ci-and-deployment.md) for the workflows, the automated Lambda deploy, and manual redeploys. +The environment is ready: lab operators can install watchers and start uploading. Point them at [Set up an instrument](https://datahub.arcadiascience.com/docs/set-up-an-instrument). Every subsequent deploy — web app, migrations, and Lambda — runs through CI; see [CI and deployment](ci-and-deployment.md) for the workflows, the automated Lambda deploy, and manual redeploys. diff --git a/developer-docs/local-development.md b/developer-docs/local-development.md index 33b574c1..4f7f12b1 100644 --- a/developer-docs/local-development.md +++ b/developer-docs/local-development.md @@ -226,5 +226,5 @@ The same builders back the integration test harness in [web/tests/integration/he - [Getting started](getting-started.md) — full setup with real Google OAuth and AWS credentials. - [Architecture](architecture.md) — system overview and data flow. -- [REST API](https://datahub.arcadiascience.com/docs/api-reference) — endpoint reference for the seeded PAT. -- [MCP server](https://datahub.arcadiascience.com/docs/mcp-server) — Model Context Protocol tools at `/api/v1/mcp`. +- [REST API](https://datahub.arcadiascience.com/docs/api) — on-ramp and generated endpoints for the seeded PAT. +- [MCP](https://datahub.arcadiascience.com/docs/mcp) — Model Context Protocol tools at `/api/v1/mcp`. diff --git a/developer-docs/watcher.md b/developer-docs/watcher.md index 70876437..aa9ada20 100644 --- a/developer-docs/watcher.md +++ b/developer-docs/watcher.md @@ -30,7 +30,7 @@ uv run data-hub-watcher init uv run data-hub-watcher watch ``` -For lab-PC installs (PyPI), see [Installing a watcher](https://datahub.arcadiascience.com/docs/installing-a-watcher). For releasing new versions and how the in-place upgrade flow works (CLI `self-update` and the Windows-service auto-updater), see [Upgrading the watcher](https://datahub.arcadiascience.com/docs/upgrading-the-watcher). +For lab-PC installs (PyPI), see [Install and update the watcher](https://datahub.arcadiascience.com/docs/install-the-watcher). For cutting releases and advertising them to the fleet, see [Roll out watcher releases](https://datahub.arcadiascience.com/docs/watcher-releases). ## Commands @@ -71,7 +71,7 @@ While running: - **Upload worker** (manual mode only) polls the server's upload queue on its own long-lived thread every 60 seconds, decoupled from the heartbeat so a slow or large upload can't delay heartbeats and make a busy watcher look offline. On shutdown it is stopped and joined before the state DB is closed. Auto mode has no worker: uploads run on the monitor's stability-checker thread via the run detector's upload callback. - **Heartbeat loop** sends periodic heartbeats (every 60 seconds) to the API. The payload includes the watcher version, instrument ID, watch directory, upload mode, per-interval activity counters, and process uptime; a final `status="stopped"` heartbeat is sent on graceful shutdown. - **Event reporter** batches and flushes lifecycle events (started, stopped, file uploaded, errors) to the API. See [Observability](#observability) for the full taxonomy. -- **Auto-updater** runs from the same heartbeat tick on every platform — not only Windows services. It polls `GET /watchers/:id/update-check` roughly hourly and applies new releases when the watcher has been idle long enough not to clobber an in-flight run. The full activity-window guard, mandatory-update behavior, and rollback flow are documented in [Upgrading the watcher](https://datahub.arcadiascience.com/docs/upgrading-the-watcher); auto-update is hard-disabled in the `preview` environment. +- **Auto-updater** runs from the same heartbeat tick on every platform — not only Windows services. It polls `GET /watchers/:id/update-check` roughly hourly and applies new releases when the watcher has been idle long enough not to clobber an in-flight run. The full activity-window guard, mandatory-update behavior, and rollback flow are documented in [Roll out watcher releases](https://datahub.arcadiascience.com/docs/watcher-releases); auto-update is hard-disabled in the `preview` environment. Use `--dry-run` to validate config and preview what would happen without starting the monitor. @@ -126,7 +126,7 @@ Manage the watcher as a Windows service: ### `self-update` -Checks the API for a newer published version and runs the appropriate `uv tool install --reinstall` (or `pip install -U`) subprocess in place. See [Upgrading the watcher](https://datahub.arcadiascience.com/docs/upgrading-the-watcher) for the supported install methods, the activity-window guard, mandatory updates, and rollback flow. +Checks the API for a newer published version and runs the appropriate `uv tool install --reinstall` (or `pip install -U`) subprocess in place. See [Install and update the watcher](https://datahub.arcadiascience.com/docs/install-the-watcher#manual-update-self-update) for supported install methods, and [Roll out watcher releases](https://datahub.arcadiascience.com/docs/watcher-releases) for the activity-window guard, mandatory updates, and rollback flow. ## Configuration diff --git a/watcher/README.md b/watcher/README.md index d0e21c9d..e6eeb0e2 100644 --- a/watcher/README.md +++ b/watcher/README.md @@ -22,7 +22,7 @@ data-hub-watcher self-update # check for and apply package updates data-hub-watcher service install # Windows: install as a service ``` -See [the operator guide](https://datahub.arcadiascience.com/docs/installing-a-watcher) for the full setup walk-through, configuration reference, and troubleshooting. +See [the operator guide](https://datahub.arcadiascience.com/docs/install-the-watcher) for the full setup walk-through, configuration reference, and troubleshooting. ## License diff --git a/watcher/pyproject.toml b/watcher/pyproject.toml index 47eb6ee2..cc329dc0 100644 --- a/watcher/pyproject.toml +++ b/watcher/pyproject.toml @@ -42,7 +42,7 @@ Homepage = "https://github.com/Arcadia-Science/data-hub" # the PyPI page actually needs. The developer-facing `developer-docs/watcher.md` # describes the editable-checkout workflow and is reachable from the # repository link below. -Documentation = "https://datahub.arcadiascience.com/docs/installing-a-watcher" +Documentation = "https://datahub.arcadiascience.com/docs/install-the-watcher" Repository = "https://github.com/Arcadia-Science/data-hub" Issues = "https://github.com/Arcadia-Science/data-hub/issues" diff --git a/web/app/settings/members/page.tsx b/web/app/settings/members/page.tsx index 9f9631b8..2fe54c4a 100644 --- a/web/app/settings/members/page.tsx +++ b/web/app/settings/members/page.tsx @@ -53,7 +53,7 @@ export default async function MembersPage() { } return ( - +

Members

diff --git a/web/components/settings/settings-page-content.tsx b/web/components/settings/settings-page-content.tsx index fc92c08b..ad5b558c 100644 --- a/web/components/settings/settings-page-content.tsx +++ b/web/components/settings/settings-page-content.tsx @@ -1,11 +1,16 @@ +import { cn } from "@/lib/utils"; + /** * Constrains settings page body copy and cards to two-thirds of the layout * container width. Access Tokens keeps full width because its table needs the space. + * Members uses a slightly wider `className` so the four-column table breathes. */ export function SettingsPageContent({ children, + className, }: { children: React.ReactNode; + className?: string; }) { - return
{children}
; + return
{children}
; } diff --git a/web/components/tokens/create-token-dialog.tsx b/web/components/tokens/create-token-dialog.tsx index 8073a8fd..75c0e669 100644 --- a/web/components/tokens/create-token-dialog.tsx +++ b/web/components/tokens/create-token-dialog.tsx @@ -103,9 +103,6 @@ function computeExpiresAt(days: string): string | undefined { } function userLabel(user: WorkspaceUser): string { - if (user.name && user.email) { - return `${user.name} (${user.email})`; - } return user.name ?? user.email ?? user.id; }