feat(cli): port recent opensrc features (fetch command, lockfile parsers, GITHUB_TOKEN auth)

CLAUDE.md

@@ -61,10 +61,10 @@ node packages/cli/dist/index.js docs add <spec> -s <source> [options]
 - `getSource(type)` (`packages/cli/src/sources/index.ts:54`) returns a `DocSource`; pass config to `DocSource.fetch(options)`, not to `getSource`. It is a single-arg factory.
 - `parseSpec` lives in `packages/cli/src/spec.ts` and returns a discriminated union (`npm` / `github` / `unknown`). It is the single source of truth for translating an `ask.json` spec string into a library slug — do not reintroduce the legacy private `parseSpec(spec) -> { name, version }` from the old `index.ts`.
 - In Nuxt Content v3 server routes, `queryCollection` is auto-imported — do NOT use `import { queryCollection } from '#content/server'` (that alias does not exist and breaks Cloudflare Pages build). Use the bare function or `import { queryCollection } from '@nuxt/content/server'`.
-- Top-level commands: `ask install | add | remove | list | docs | search | skills | src | cache` (registered in `packages/cli/src/index.ts:299`). `ask search <spec> <query>` resolves the pinned checkout via `ensureCheckout` then delegates to `csp` (code-search) for semantic search; csp is optional (probe `$CSP_BIN`→PATH, else print path + recipe and exit 0). The **legacy** `ask docs` namespace that had `add | sync | list | remove` sub-commands is gone — there is no deprecated wrapper. The **current** `ask docs <spec>` is a NEW single-shot command (`packages/cli/src/commands/docs.ts`) that prints candidate doc paths: for npm specs it walks `node_modules/<pkg>/`, then always walks the cached checkout at `<askHome>/github/<host>/<owner>/<repo>/<ref>/` and emits the root plus any `/doc/i` subdirs. It shares `ensureCheckout` with `ask src` and does NOT go through `runInstall`, so strict ref validation does not apply — unqualified github specs default `ref = 'main'` and clone the branch directly. `ask install` is `postinstall`-friendly: per-entry failures emit a warning and exit code is always 0 (FR-10).
+- Top-level commands: `ask install | add | remove | list | docs | fetch | search | skills | src | cache` (registered in `packages/cli/src/index.ts`). `ask fetch <spec...>` (ported from opensrc's `opensrc fetch`) warms the checkout cache via `ensureCheckout` without printing paths — supports multiple specs, `-q/--quiet`, per-spec error reporting with non-zero exit at the end. `ask search <spec> <query>` resolves the pinned checkout via `ensureCheckout` then delegates to `csp` (code-search) for semantic search; csp is optional (probe `$CSP_BIN`→PATH, else print path + recipe and exit 0). The **legacy** `ask docs` namespace that had `add | sync | list | remove` sub-commands is gone — there is no deprecated wrapper. The **current** `ask docs <spec>` is a NEW single-shot command (`packages/cli/src/commands/docs.ts`) that prints candidate doc paths: for npm specs it walks `node_modules/<pkg>/`, then always walks the cached checkout at `<askHome>/github/<host>/<owner>/<repo>/<ref>/` and emits the root plus any `/doc/i` subdirs. It shares `ensureCheckout` with `ask src` and does NOT go through `runInstall`, so strict ref validation does not apply — unqualified github specs default `ref = 'main'` and clone the branch directly. `ask install` is `postinstall`-friendly: per-entry failures emit a warning and exit code is always 0 (FR-10).
 - `ensureCheckout` (shared by `ask src` and `ask docs`) always sets `skipDocExtraction: true` on its `GithubSourceOptions`. The source's `extractDocsFromDir` scan is a correctness precondition for `runInstall` (materializing `.ask/docs/<name>@<version>/` requires a real file list), but for `ask src`/`ask docs` it is actively wrong — those commands walk the checkout themselves (`findDocLikePaths`) and must not crash on repos without a conventional `docs/` folder (regression: `gitbutlerapp/gitbutler`). Keep the flag opt-in; never set it inside `runInstall`.
 - `cpDirAtomic` (`packages/cli/src/store/index.ts`) passes `verbatimSymlinks: true` to `fs.cpSync`. Without it, Node resolves relative symlinks against the SOURCE directory at copy time, baking an absolute path pointing at the soon-deleted `ask-gh-clone-*` tmp dir into the store entry — every later `verifyEntry` call then hits ENOENT reading the now-broken link (observed on gitbutler's `claude.md`/`CLAUDE.md` → `AGENTS.md` links). `hashDir` walks every file via `collectFiles` + `readFileSync`, so it has no tolerance for a broken symlink. Keep the flag.
-- PM-driven entries (`{ "spec": "npm:<pkg>" }`) get their version from the project's lockfile in priority order: `bun.lock → package-lock.json → pnpm-lock.yaml → yarn.lock → package.json` (range fallback). The chain lives in `packages/cli/src/lockfiles/index.ts:npmEcosystemReader`. Standalone github entries (`{ "spec": "github:owner/repo", "ref": "v1.2.3" }`) require an explicit `ref` enforced at the schema layer — there is no implicit default branch.
+- PM-driven entries (`{ "spec": "npm:<pkg>" }`) get their version from the project's lockfile in priority order: `bun.lock → package-lock.json → pnpm-lock.yaml → yarn.lock → package.json` (range fallback). The chain lives in `packages/cli/src/lockfiles/index.ts:npmEcosystemReader`. The pnpm/yarn readers are format-aware parsers ported from opensrc (indent-aware stack parser with BFS transitive resolution for pnpm v5–v9; block-based parser for yarn v1 + Berry) — do not reintroduce the old regex parsers, they false-match pnpm peer-dep suffixes and yarn multi-specifier headers. All readers (incl. the `package.json` fallback) skip protocol versions (`workspace:*`, `link:`, `file:`, ...) via `isRegistryVersion`. Standalone github entries (`{ "spec": "github:owner/repo", "ref": "v1.2.3" }`) require an explicit `ref` enforced at the schema layer — there is no implicit default branch.
 - A PreToolUse Bash hook blocks shell commands when the current commit has no recorded review. Run `/review:code-review` (or `/review:run-cubic` / `/review:run-gemini`) and let it call `save-review-state.sh` before further bash.
 - `apps/registry/` uses `vercel.ts` (programmatic config via `@vercel/config` devDep), not `vercel.json`. Use `git.deploymentEnabled` (not deprecated `github.enabled`) to control auto-deploy.
 - Track artifacts live under `.please/docs/tracks/active/{slug}-{YYYYMMDD}/` with `spec.md`, `plan.md`, `metadata.json`. Append a JSON line to `.please/docs/tracks.jsonl` when creating a track.
@@ -90,13 +90,14 @@ node packages/cli/dist/index.js docs add <spec> -s <source> [options]
 - `github`-kind store entries live at `<askHome>/github/<host>/<owner>/<repo>/<tag>/` (PM-style nested layout, host hard-coded to `github.com` for MVP). Do NOT reintroduce `github/db/` or `github/checkouts/` — those are LEGACY and collapse five correctness defects structurally when removed. The shared bare-clone subsystem (`packages/cli/src/store/github-bare.ts`) is gone; each entry is an independent shallow clone via `git clone --depth 1 --branch <tag> --single-branch` with `.git/` stripped and commit SHA captured into `ResolvedEntry.commit` before the strip. Fallback chain in `cloneAtTag` (`packages/cli/src/sources/github.ts`): try ref as-is; if it does NOT start with `v`, also try `v<ref>`. Never emit `vv1.2.3`. See track `github-store-pm-unified-20260411`.
 - `ask.json` ref validation is **strict by default**: `main`/`master`/`develop`/`trunk`/`HEAD`/`latest` and any single-word ref without `.` or digit are rejected at the CLI boundary (in `runInstall` via `validateAskJsonStrict`). Schema package exports both `AskJsonSchema` (strict) and `LaxAskJsonSchema` (no refinement). `readAskJson`/`writeAskJson` ALWAYS use lax so internal readers (`listDocs`, `generateAgentsMd`, `manageIgnoreFiles`, etc.) never need to know about `--allow-mutable-ref`. CI/test paths pass `--allow-mutable-ref` to skip the strict validation call; the schema parser still reads the lax variant underneath.
 - `FetchResult.storeSubpath` (`packages/cli/src/sources/index.ts`) is populated by the github source from the entry's `docsPath`. Link/ref mode in `storage.ts:saveDocs` joins `path.join(storePath, storeSubpath ?? '')` so symlinks and AGENTS.md-rendered paths point at the docs subdirectory, not the repo root. npm/web/llms-txt leave `storeSubpath` undefined and the behaviour is unchanged.
+- Private GitHub repos: `GITHUB_TOKEN` is injected by `authenticatedCloneUrl` (`packages/cli/src/sources/github.ts`) ONLY on exact `https://github.com` host matches (opensrc#66 host-confusion fix — `github.com.evil.com` etc. never get the token). Auth happens at the exec boundary; error messages keep the token-free URL and `redactToken` scrubs git/execFileSync output. Never pass the authenticated URL into a log or thrown Error.
 - Store-hit short-circuits in both `packages/cli/src/install.ts:274` (npm) and `packages/cli/src/sources/github.ts` (github) run `verifyEntry(storeDir)` before trusting a cached entry. Failures move the entry to `<askHome>/.quarantine/<ts>-<uuid>/` via `quarantineEntry` (in `store/index.ts`) and fall through to a fresh fetch. Never reintroduce a bare `fs.existsSync(storeDir)` check without the guard.
 - `bun pm pack` resolves `workspace:*` from `bun.lock`, not `package.json` (oven-sh/bun#20477). release-please bumps `package.json` versions but `bun install` won't propagate them to the lockfile (oven-sh/bun#18906). The release workflow regenerates the lockfile (`rm bun.lock && bun install`) before packing to work around this. If you ever change the publish pipeline, ensure the lockfile is fresh.
 - `<askHome>/STORE_VERSION` is written on every `ask install` start (`writeStoreVersion` in `store/index.ts`). Current value is `"2"`. A legacy `github/db` or `github/checkouts` directory triggers a one-line warning and points at `ask cache clean --legacy`. `cacheLs` walks BOTH layouts and tags legacy entries with `legacy: true` + a `(legacy) ` key prefix.
 
 - `consola.prompt()` supports `multiselect`, `text`, `confirm`, `select` types (v3.4.2+). The multiselect return type needs `as unknown as string[]` cast because TS infers the options object type, not the value type.
 - `ask.json` `libraries` is a union `string | { spec, docsPaths }` (schema in `packages/schema/src/ask-json.ts`). The object form is emitted ONLY when the user selects a docs-path subset; the canonical "no override" form is the bare string (enforced in `entryFromSpec` and by `docsPaths.nonempty()`). Every consumer that iterates `askJson.libraries` MUST wrap entries with `specFromEntry` — the exported helpers `specFromEntry` / `docsPathsFromEntry` / `entryFromSpec` live in the same schema file and are re-exported from `packages/cli/src/schemas.ts`. `io.ts:findEntry` is the standard lookup helper (matches by exact spec → library slug → npm package name, same rules as `ask remove`).
-- `ask add`'s docs-path prompt (`packages/cli/src/commands/add.ts:runAdd`) is **offline-first**: `gatherDocsCandidates` calls `ensureCheckout({ noFetch: true })` and silently skips the cached-checkout group on `NoCacheError`. It never triggers a fresh clone. First-time `ask add <spec>` on an uncached spec gets no prompt — users must run `ask docs <spec>` first to warm the cache, then re-run `ask add`. Flags: `--docs-paths <csv>` (non-interactive override) and `--clear-docs-paths` (downgrade object entry → bare string). `CandidateGatheringError` is caught by `runAdd` → warn + persist the spec without an override.
+- `ask add`'s docs-path prompt (`packages/cli/src/commands/add.ts:runAdd`) is **offline-first**: `gatherDocsCandidates` calls `ensureCheckout({ noFetch: true })` and silently skips the cached-checkout group on `NoCacheError`. It never triggers a fresh clone. First-time `ask add <spec>` on an uncached spec gets no prompt — users must run `ask fetch <spec>` (or `ask docs <spec>`) first to warm the cache, then re-run `ask add`. Flags: `--docs-paths <csv>` (non-interactive override) and `--clear-docs-paths` (downgrade object entry → bare string). `CandidateGatheringError` is caught by `runAdd` → warn + persist the spec without an override.
 - `ask docs` (`packages/cli/src/commands/docs.ts:runDocs`) consults `readAskJson` + `findEntry` for a stored `docsPaths` override. Stored paths are resolved against both the npm root (`node_modules/<pkg>`) and `result.checkoutDir`, existing-file wins, zero-survivor case prints a stderr warning and falls through to the unfiltered walk. The prompt and the consumer both store / read paths **relative to the group root** so the entry stays portable across machines and cache wipes.
 - Monorepo packages (changesets convention) use `<pkg>@<version>` git tags, not `v<version>`. The npm resolver detects monorepos via `repository.directory` in npm registry metadata and generates `<unscoped-pkg-name>@<version>` as `fallbackRefs`. `GithubSource` tries these before the standard `v<version>` chain. As a last resort, `cloneAtTag()` probes `git ls-remote --tags` to discover matching tags dynamically. On total failure, the error message lists discovered tags so users/LLM agents can retry with `--ref <exact-tag>`.
 - When disambiguating bare names (no `:` prefix): check `input.startsWith('@')` before `OWNER_REPO_RE` — scoped npm packages like `@vercel/ai` match the `owner/repo` regex pattern and get misrouted to `github:`.

README.md

@@ -178,13 +178,20 @@ documentation**. They print absolute paths to cached source trees,
 fetching on cache miss:
 
 ```bash
-ask src <spec>     # Print the absolute path to a cached library source tree
-ask docs <spec>    # Print all candidate documentation paths from node_modules + the cached source
+ask src <spec>          # Print the absolute path to a cached library source tree
+ask docs <spec>         # Print all candidate documentation paths from node_modules + the cached source
+ask fetch <spec...>     # Warm the cache for one or more specs without printing paths (-q/--quiet)
 ```
 
-Both commands fetch on cache miss (first run) and short-circuit on cache
-hit. They share the same `~/.ask/github/github.com/<owner>/<repo>/<tag>/`
-store.
+All three commands fetch on cache miss (first run) and short-circuit on
+cache hit. They share the same
+`~/.ask/github/github.com/<owner>/<repo>/<tag>/` store. `ask fetch` is
+the prefetch counterpart for setup scripts — it also warms the cache
+that `ask add`'s offline-first docs-path prompt reads from.
+
+Private GitHub repositories are supported by setting `GITHUB_TOKEN` in
+the environment; the token is attached only to exact `https://github.com`
+remotes.
 
 ```bash
 # Print the absolute path to React's source tree (fetches on first run)

packages/cli/src/commands/ensure-checkout.ts

@@ -33,6 +33,12 @@ export interface EnsureCheckoutResult {
    * Undefined for github/pypi/pub/etc. specs.
    */
   npmPackageName?: string
+  /**
+   * True when the checkout was already in the store (no network fetch
+   * happened). `ask fetch` uses this to report "fetched" vs "already
+   * cached" per spec.
+   */
+  fromCache: boolean
 }
 
 /**
@@ -41,7 +47,12 @@ export interface EnsureCheckoutResult {
  */
 export interface EnsureCheckoutDeps {
   askHome?: string
-  fetcher?: { fetch: (opts: SourceConfig) => Promise<FetchResult> }
+  /**
+   * The result may be undefined: test seams simulate a successful fetch
+   * by materializing the checkout dir without building a `FetchResult`.
+   * Downstream reads must stay null-safe (`fetchResult?.…`).
+   */
+  fetcher?: { fetch: (opts: SourceConfig) => Promise<FetchResult | undefined> }
   lockfileReader?: { read: (name: string, projectDir: string) => { version: string } | null }
   resolverFor?: (ecosystem: string) => {
     resolve: (name: string, version: string) => Promise<{
@@ -205,7 +216,7 @@ export async function ensureCheckout(
 
   // 5. Cache hit short-circuit
   if (fs.existsSync(checkoutDir)) {
-    return { parsed, owner, repo, ref, resolvedVersion, checkoutDir, npmPackageName }
+    return { parsed, owner, repo, ref, resolvedVersion, checkoutDir, npmPackageName, fromCache: true }
   }
 
   // 6. Cache miss + noFetch → throw
@@ -253,5 +264,8 @@ export async function ensureCheckout(
   // the path downstream tools actually index.
   const actualRef = resolvedCheckoutDir === checkoutDir ? ref : path.basename(resolvedCheckoutDir)
 
-  return { parsed, owner, repo, ref: actualRef, resolvedVersion, checkoutDir: resolvedCheckoutDir, npmPackageName }
+  // `GithubSource.fetch` can satisfy the request from its own store-hit
+  // path (a ref-candidate variant like `v<ref>` or `master`) with zero
+  // network I/O — trust its `fromStoreCache` over our primary-key miss.
+  return { parsed, owner, repo, ref: actualRef, resolvedVersion, checkoutDir: resolvedCheckoutDir, npmPackageName, fromCache: fetchResult?.fromStoreCache ?? false }
 }