Make watcher API base URL configurable per environment

developer-docs/watcher.md

@@ -43,7 +43,7 @@ All commands accept two global flags:
 
 Interactive setup wizard that:
 
-1. Prompts for the environment (`staging`, `production`, or `preview`). Choosing `preview` also prompts for a custom API base URL.
+1. Prompts for the environment (`staging`, `production`, or `preview`) and the Data Hub API base URL for it. There is no built-in default URL — each environment points at whatever deployment you host — and the URL is saved per environment so switching back later reuses it without re-prompting.
 2. Prompts for an API key (or reads `DATA_HUB_API_KEY` from the environment). The key is saved to a per-environment file at `~/.data-hub/.env.<environment>` (e.g. `.env.staging`), so switching between environments later doesn't require re-entering it.
 3. Fetches existing instruments from the API or registers a new one.
 4. Prompts for the watch directory, file patterns, run detection pattern, stability period, and upload mode.
@@ -137,7 +137,8 @@ The config file lives at `~/.data-hub/config.yaml` by default. Override with `--
 ```yaml
 version: 1
 environment: production          # "staging", "production", or "preview"
-api_base_url: null               # required when environment is "preview"
+api_base_urls:                   # one API base URL per environment (no default)
+  production: https://datahub.example.com/api/v1
 watcher_ids:                     # one registration id per environment
   production: <assigned-by-api>
 initial_scan: null               # null (default), "full", or "new-only"
@@ -177,17 +178,21 @@ A config written by an older watcher used a single top-level `watcher_id`. It is
 
 ### Switching environments
 
-A single PC can move between `staging`, `production`, and `preview` and back. Because staging and production are separate Data Hub deployments with separate databases, each holds its own watcher registration — `watcher_ids` stores one id per environment, and the credentials live in per-environment `~/.data-hub/.env.<environment>` files.
+A single PC can move between `staging`, `production`, and `preview` and back. Because these are separate Data Hub deployments with separate databases, each holds its own watcher registration and its own API base URL — `watcher_ids` stores one id per environment, `api_base_urls` stores one URL per environment, and the credentials live in per-environment `~/.data-hub/.env.<environment>` files.
+
+The first switch to an environment needs its URL via `--api-base-url`; later switches reuse the stored value, so you never re-enter it.
 
 Switch with:
 
 ```sh
-# Switch to staging (reuses a stored registration, or registers one).
-uv run data-hub-watcher config set-environment staging
+# First switch to an environment: provide its base URL (stored for next time).
+uv run data-hub-watcher config set-environment staging \
+  --api-base-url https://datahub-staging.example.com/api/v1
 
-# Point at a preview deployment (the base URL is required).
-uv run data-hub-watcher config set-environment preview \
-  --api-base-url https://data-hub-git-my-branch.vercel.app/api/v1
+# Subsequent switches reuse the stored URL — no flag needed.
+uv run data-hub-watcher config set-environment production \
+  --api-base-url https://datahub.example.com/api/v1
+uv run data-hub-watcher config set-environment staging
 ```
 
 `config edit` also re-prompts for the environment and runs the same switch flow when it changes. Useful flags: `--api-key` (otherwise the key is read from the env file or prompted), `--show-key`, and `--no-register` (fail instead of registering a new watcher if none is stored for the target).

watcher/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-hub-watcher"
-version = "0.4.0"
+version = "0.5.0"
 description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
 readme = "README.md"
 requires-python = ">=3.12"

watcher/src/data_hub_watcher/cli.py

@@ -15,7 +15,6 @@
 from data_hub_watcher.api_client import ApiError, DataHubClient
 from data_hub_watcher.config_io import config_checksum, load_config, save_config
 from data_hub_watcher.constants import (
-    API_URLS,
     DEFAULT_CONFIG_DIR,
     DEFAULT_STABILITY_PERIOD_SECONDS,
     RUN_DETECTION_PRESETS,
@@ -145,13 +144,16 @@ def _clean_api_key(value: str) -> str:
 def _make_client(
     environment: str, api_key: str | None = None, api_base_url: str | None = None
 ) -> DataHubClient:
-    if environment == "preview":
-        if not api_base_url:
-            raise click.ClickException("api_base_url is required for the 'preview' environment.")
-        base_url = api_base_url
-    else:
-        base_url = API_URLS[environment]
-    return DataHubClient(base_url, api_key=api_key)
+    # Every environment gets its URL from config now — there is no baked-in
+    # default — so a missing URL is a hard error rather than a silent fall
+    # through to someone else's server.
+    if not api_base_url:
+        raise click.ClickException(
+            f"No API base URL is configured for the '{environment}' environment. "
+            "Set one with `data-hub-watcher init` or "
+            f"`data-hub-watcher config set-environment {environment} --api-base-url <url>`."
+        )
+    return DataHubClient(api_base_url, api_key=api_key)
 
 
 def _load_and_client(ctx: click.Context) -> tuple[WatcherConfig, DataHubClient, Path]:
@@ -205,12 +207,15 @@ def init(ctx: click.Context, show_key: bool) -> None:
         type=click.Choice(list(SUPPORTED_ENVIRONMENTS), case_sensitive=False),
     )
 
-    api_base_url: str | None = None
-    if environment == "preview":
-        raw_url: str = click.prompt(
-            "Preview deployment base URL (e.g. https://data-hub-git-my-branch.vercel.app/api/v1)"
-        )
-        api_base_url = raw_url.rstrip("/")
+    # Every environment needs its own Data Hub API base URL; there is no
+    # baked-in default. Preview points at an ephemeral branch deploy, the
+    # others at the operator's own staging/production host.
+    url_example = (
+        "https://data-hub-git-my-branch.vercel.app/api/v1"
+        if environment == "preview"
+        else "https://datahub.example.com/api/v1"
+    )
+    api_base_url: str = click.prompt(f"Data Hub API base URL (e.g. {url_example})").rstrip("/")
 
     # 2. API key — overlay any existing per-environment env file so the user
     # doesn't have to re-enter a key they've already saved for this target.
@@ -332,7 +337,7 @@ def init(ctx: click.Context, show_key: bool) -> None:
     config = WatcherConfig(
         version=1,
         environment=environment,
-        api_base_url=api_base_url,
+        api_base_urls={environment: api_base_url},
         watcher_ids={environment: watcher_id},
         instrument=InstrumentConfig(
             id=selected.id,
@@ -576,20 +581,30 @@ def _switch_environment(
     inst = cfg.instrument
     config_dir = path.parent
 
-    if target == "preview":
-        if not api_base_url:
-            raise click.ClickException("--api-base-url is required when switching to 'preview'.")
-        api_base_url = api_base_url.rstrip("/")
+    # Resolve the target environment's API base URL: an explicit --api-base-url
+    # wins, otherwise reuse the URL already stored for that environment so
+    # switching back never re-prompts. With neither available we can't build a
+    # client, so fail before any network or credential work.
+    stored_url = cfg.api_base_urls.get(target)
+    if api_base_url:
+        target_url = api_base_url.rstrip("/")
+    elif stored_url:
+        target_url = stored_url
+    else:
+        raise click.ClickException(
+            f"--api-base-url is required to switch to '{target}': "
+            "no API base URL is stored for it yet."
+        )
 
     # A preview redeploy points at a new database server-side, so the stored
     # registration won't exist and local state must be reset. Compare against
     # the URL the DB was seeded against (`meta`), falling back to the config.
     preview_redeploy = False
-    if target == "preview" and cfg.environment == "preview" and api_base_url is not None:
-        seeded_url = _preview_seed_url(config_dir) or cfg.api_base_url
-        preview_redeploy = seeded_url is not None and api_base_url != seeded_url
+    if target == "preview" and cfg.environment == "preview":
+        seeded_url = _preview_seed_url(config_dir) or stored_url
+        preview_redeploy = seeded_url is not None and target_url != seeded_url
 
-    if cfg.environment == target and not preview_redeploy:
+    if cfg.environment == target and target_url == stored_url and not preview_redeploy:
         click.echo(f"Already on environment '{target}'.")
         return cfg
 
@@ -605,7 +620,7 @@ def _switch_environment(
             api_key = _clean_api_key(click.prompt("DATA_HUB_API_KEY", hide_input=not show_key))
             save_api_key(api_key, target)
 
-    client = _make_client(target, api_key=api_key, api_base_url=api_base_url)
+    client = _make_client(target, api_key=api_key, api_base_url=target_url)
     try:
         client.list_instruments()
     except ApiError as exc:
@@ -640,10 +655,13 @@ def _switch_environment(
         watcher_ids[target] = watcher_id
         click.echo(f"Registered watcher for {target}: {watcher_id}")
 
+    # Preserve every other environment's stored URL; only (re)set the target's.
+    api_base_urls = {**cfg.api_base_urls, target: target_url}
+
     new_cfg = WatcherConfig(
         version=cfg.version,
         environment=target,  # type: ignore[arg-type]  # validated by click.Choice
-        api_base_url=api_base_url if target == "preview" else None,
+        api_base_urls=api_base_urls,
         watcher_ids=watcher_ids,
         initial_scan=cfg.initial_scan,
         instrument=inst,
@@ -654,10 +672,10 @@ def _switch_environment(
     # Record which preview deployment this state DB was seeded against; the
     # next switch reads it back via `_preview_seed_url` to decide whether the
     # URL changed and the DB must be reset (see `preview_redeploy`).
-    if target == "preview" and api_base_url is not None:
+    if target == "preview":
         try:
             db = StateDB(state_db_path(config_dir, target))
-            db.set_meta(PREVIEW_SEED_URL_META_KEY, api_base_url)
+            db.set_meta(PREVIEW_SEED_URL_META_KEY, target_url)
             db.close()
         except Exception:
             logger.debug("Could not record preview deployment URL", exc_info=True)
@@ -801,7 +819,12 @@ def config_validate(ctx: click.Context) -> None:
     "environment",
     type=click.Choice(list(SUPPORTED_ENVIRONMENTS), case_sensitive=False),
 )
-@click.option("--api-base-url", default=None, help="Required when environment is 'preview'.")
+@click.option(
+    "--api-base-url",
+    default=None,
+    help="API base URL for the target environment. Required the first time you "
+    "switch to an environment; reused from config on later switches.",
+)
 @click.option("--api-key", default=None, help="API key for the target environment.")
 @click.option(
     "--show-key",
@@ -845,22 +868,20 @@ def config_edit(ctx: click.Context) -> None:
 
     # Re-prompt for environment first; a change delegates to the full switch
     # flow (re-registration, key resolution, state reset) before the rest of
-    # the edit runs against the freshly-saved config.
+    # the edit runs against the freshly-saved config. The switch reuses the
+    # target env's stored URL, prompting via --api-base-url only when none is
+    # saved yet, so no URL prompt is needed here.
     target_env = click.prompt(
         "Environment",
         type=click.Choice(list(SUPPORTED_ENVIRONMENTS), case_sensitive=False),
         default=cfg.environment,
     )
-    preview_url: str | None = None
-    if target_env == "preview":
-        preview_url = click.prompt(
-            "Preview deployment base URL",
-            default=cfg.api_base_url,
-        )
-    if target_env != cfg.environment or (
-        target_env == "preview" and preview_url is not None and preview_url != cfg.api_base_url
-    ):
-        cfg = _switch_environment(path, cfg, target_env, api_base_url=preview_url)
+    if target_env != cfg.environment:
+        stored_url = cfg.api_base_urls.get(target_env)
+        api_base_url = None
+        if not stored_url:
+            api_base_url = click.prompt(f"Data Hub API base URL for {target_env}").rstrip("/")
+        cfg = _switch_environment(path, cfg, target_env, api_base_url=api_base_url)
 
     inst = cfg.instrument
 
@@ -898,7 +919,7 @@ def config_edit(ctx: click.Context) -> None:
     new_config = WatcherConfig(
         version=cfg.version,
         environment=cfg.environment,
-        api_base_url=cfg.api_base_url,
+        api_base_urls=cfg.api_base_urls,
         watcher_ids=cfg.watcher_ids,
         initial_scan=cfg.initial_scan,
         instrument=InstrumentConfig(

watcher/src/data_hub_watcher/constants.py

@@ -7,11 +7,6 @@
 
 from dotenv import load_dotenv
 
-API_URLS: dict[str, str] = {
-    "staging": "https://data-hub-env-staging-arcadia-science.vercel.app/api/v1",
-    "production": "https://data-hub.arcadiascience.com/api/v1",
-}
-
 
 def _read_watcher_version() -> str:
     # Resolve the installed distribution version once at import time so the

watcher/src/data_hub_watcher/models.py

@@ -72,7 +72,13 @@ def _validate_directory(cls, v: Path) -> Path:
 class WatcherConfig(BaseModel):
     version: Literal[1]
     environment: Literal["staging", "production", "preview"]
-    api_base_url: str | None = None
+    # One API base URL per environment, keyed by the `environment` literal, so
+    # a single PC can switch between staging/production/preview without
+    # re-entering the URL each time. There is no baked-in default: every
+    # environment a host actually uses must have its URL set (prompted at
+    # `init`, reused on switch). The active environment's URL is exposed via
+    # the `api_base_url` property below.
+    api_base_urls: dict[str, str] = Field(default_factory=dict)
     # One watcher_id per environment so a single PC can switch between
     # staging/production/preview without losing the registration it already
     # holds in each. Keyed by the `environment` literal.
@@ -85,22 +91,39 @@ class WatcherConfig(BaseModel):
 
     @model_validator(mode="before")
     @classmethod
-    def _migrate_legacy_watcher_id(cls, data: Any) -> Any:
-        # Pre-multi-env configs (and pre-multi-env constructor calls) carry a
-        # single top-level `watcher_id`. Lift it into `watcher_ids` under the
-        # active environment so on-disk configs migrate transparently on load.
-        if isinstance(data, dict) and "watcher_id" in data and "watcher_ids" not in data:
+    def _migrate_legacy_fields(cls, data: Any) -> Any:
+        # Pre-multi-env configs (and pre-multi-env constructor calls) carry
+        # single top-level `watcher_id` / `api_base_url` scalars. Lift them
+        # into their per-environment maps under the active environment so
+        # on-disk configs (and legacy `api_base_url=` kwargs) migrate
+        # transparently on load.
+        if not isinstance(data, dict):
+            return data
+        env = data.get("environment")
+        if "watcher_id" in data and "watcher_ids" not in data:
             legacy = data.pop("watcher_id")
-            env = data.get("environment")
             if legacy and env:
                 data["watcher_ids"] = {env: legacy}
+        if "api_base_url" in data and "api_base_urls" not in data:
+            legacy_url = data.pop("api_base_url")
+            if legacy_url and env:
+                data["api_base_urls"] = {env: legacy_url}
+        elif "api_base_url" in data:
+            # Both the scalar and the map were supplied; drop the scalar so it
+            # isn't flagged as an unexpected field.
+            data.pop("api_base_url")
         return data
 
     @property
     def watcher_id(self) -> str | None:
         """The watcher_id registered for the active environment, if any."""
         return self.watcher_ids.get(self.environment)
 
+    @property
+    def api_base_url(self) -> str | None:
+        """The API base URL configured for the active environment, if any."""
+        return self.api_base_urls.get(self.environment)
+
     def resolve_initial_scan(self) -> Literal["full", "new-only"]:
         """Effective initial-scan mode, defaulting by environment when unset.
 
@@ -112,12 +135,6 @@ def resolve_initial_scan(self) -> Literal["full", "new-only"]:
             return self.initial_scan
         return "full" if self.environment == "production" else "new-only"
 
-    @model_validator(mode="after")
-    def _validate_preview_url(self) -> WatcherConfig:
-        if self.environment == "preview" and not self.api_base_url:
-            raise ValueError("api_base_url is required when environment is 'preview'")
-        return self
-
     @model_validator(mode="after")
     def _emit_warnings(self) -> WatcherConfig:
         # Surface common misconfigurations as warnings at load time rather

watcher/src/data_hub_watcher/service.py

@@ -594,7 +594,6 @@ def _run_service_loop(stop_event: threading.Event, sm: Any) -> None:
     from data_hub_watcher.api_client import ApiError, DataHubClient
     from data_hub_watcher.config_io import load_config
     from data_hub_watcher.constants import (
-        API_URLS,
         env_file_path,
         resolve_state_db_path,
     )
@@ -655,14 +654,19 @@ def _run_service_loop(stop_event: threading.Event, sm: Any) -> None:
     cfg = load_config(path)
     inst = cfg.instrument
 
-    if cfg.environment == "preview":
-        # WatcherConfig's model validator guarantees api_base_url is
-        # set whenever environment is "preview"; the assertion is here
-        # to make that invariant visible to pyright.
-        assert cfg.api_base_url is not None
-        base_url = cfg.api_base_url
-    else:
-        base_url = API_URLS[cfg.environment]
+    base_url = cfg.api_base_url
+    if not base_url:
+        # No baked-in default exists anymore; a config without a URL for the
+        # active environment can't build a client. Fail loudly so the operator
+        # re-runs `config set-environment` rather than the service silently
+        # looping on connection errors.
+        logger.error(
+            "No API base URL configured for environment %r. "
+            "Re-run 'data-hub-watcher config set-environment %s --api-base-url <url>'.",
+            cfg.environment,
+            cfg.environment,
+        )
+        raise SystemExit(1)
     client = DataHubClient(base_url)
 
     # Step 1: Check instrument status (mirrors CLI watch startup)

watcher/tests/test_cli_self_update.py

@@ -43,6 +43,7 @@ def _make_config(tmp_path: Path) -> WatcherConfig:
     return WatcherConfig(
         version=1,
         environment="staging",
+        api_base_urls={"staging": "https://staging.example.test/api/v1"},
         watcher_ids={"staging": "w-test"},
         instrument=InstrumentConfig(
             id="test-instrument",