diff --git a/.please/docs/tracks.jsonl b/.please/docs/tracks.jsonl index 9aa203e..1ffff67 100644 --- a/.please/docs/tracks.jsonl +++ b/.please/docs/tracks.jsonl @@ -2,4 +2,4 @@ {"id":"parallelize-sync-fetches-20260407","type":"chore","status":"review","phase":"review","issue":"#2","created":"2026-04-07","section":"completed","pr":"#8"} {"id":"cli-shorthand-20260407","type":"feature","status":"review","phase":"finalize","issue":"#4","pr":"#10","created":"2026-04-07","section":"completed"} {"id":"registry-meta-20260407","type":"feature","status":"in_progress","phase":"implement","issue":"#5","created":"2026-04-07","section":"active"} -{"id":"ecosystem-resolvers-20260407","type":"feature","status":"planned","phase":"spec","issue":"#6","created":"2026-04-07","section":"active"} +{"id":"ecosystem-resolvers-20260407","type":"feature","status":"in_progress","phase":"implement","issue":"#6","created":"2026-04-07","section":"active"} diff --git a/.please/docs/tracks/active/ecosystem-resolvers-20260407/metadata.json b/.please/docs/tracks/completed/ecosystem-resolvers-20260407/metadata.json similarity index 56% rename from .please/docs/tracks/active/ecosystem-resolvers-20260407/metadata.json rename to .please/docs/tracks/completed/ecosystem-resolvers-20260407/metadata.json index 465bfee..4b53581 100644 --- a/.please/docs/tracks/active/ecosystem-resolvers-20260407/metadata.json +++ b/.please/docs/tracks/completed/ecosystem-resolvers-20260407/metadata.json @@ -1,10 +1,10 @@ { "track_id": "ecosystem-resolvers-20260407", "type": "feature", - "status": "planned", + "status": "review", "created_at": "2026-04-07T00:00:00Z", - "updated_at": "2026-04-07T00:00:00Z", + "updated_at": "2026-04-08T00:00:00Z", "issue": "#6", - "pr": "https://github.com/pleaseai/ask/pull/7", + "pr": "#13", "project": "" } diff --git a/.please/docs/tracks/active/ecosystem-resolvers-20260407/plan.md b/.please/docs/tracks/completed/ecosystem-resolvers-20260407/plan.md similarity index 57% rename from .please/docs/tracks/active/ecosystem-resolvers-20260407/plan.md rename to .please/docs/tracks/completed/ecosystem-resolvers-20260407/plan.md index cee8d93..4d12b03 100644 --- a/.please/docs/tracks/active/ecosystem-resolvers-20260407/plan.md +++ b/.please/docs/tracks/completed/ecosystem-resolvers-20260407/plan.md @@ -32,15 +32,15 @@ ask docs add npm:lodash ## Tasks -- **T-1** [impl] `EcosystemResolver` interface, `parseRepoUrl` utility, and unit tests -- **T-2** [impl] npm resolver — registry API + dist-tags + semver resolution -- **T-3** [impl] pypi resolver — extract `project_urls`, PEP 440 handling -- **T-4** [impl] pub resolver — extract `pubspec.repository` -- **T-5** [test] Unit tests for each resolver (mocked fetch) -- **T-6** [impl] Wire resolvers into the `add` command — registry-miss fallback -- **T-7** [test] End-to-end smoke — `ask docs add npm:lodash`, `pub:riverpod` -- **T-8** [docs] Update README -- **T-9** [chore] Regression — confirm registry-hit `npm:next` still works +- [x] **T-1** [impl] `EcosystemResolver` interface, `parseRepoUrl` utility, and unit tests +- [x] **T-2** [impl] npm resolver — registry API + dist-tags + semver resolution +- [x] **T-3** [impl] pypi resolver — extract `project_urls`, PEP 440 handling +- [x] **T-4** [impl] pub resolver — extract `pubspec.repository` +- [x] **T-5** [test] Unit tests for each resolver (mocked fetch) +- [x] **T-6** [impl] Wire resolvers into the `add` command — registry-miss fallback +- [x] **T-7** [test] End-to-end smoke — `ask docs add npm:lodash`, `pub:riverpod` +- [x] **T-8** [docs] Update README +- [x] **T-9** [chore] Regression — confirm registry-hit `npm:next` still works ## Risks @@ -48,6 +48,29 @@ ask docs add npm:lodash - Git tag conventions differ (`v1.0.0` vs `1.0.0`) → try both, fall back on github 404 - Semver range parsing pulls in a new dependency — consider adding `semver` +## Outcomes & Retrospective + +### What Was Shipped +- EcosystemResolver interface + factory (`getResolver`) for npm, pypi, pub +- `parseRepoUrl` utility for normalizing GitHub URLs from varied formats +- Semver range resolution for npm (`^15` → latest 15.x.x) via `semver` package +- Git ref fallback (`v{version}` → `{version}` or vice versa) +- Resolver fallback wired into `add` command on registry miss +- 146 tests passing, lint + tsc clean + +### What Went Well +- Clean TDD cycle — tests defined expected behavior, implementation followed +- Spec compliance check caught semver range gap early before merge +- Resolvers are fully decoupled from sources, testable with mock fetch + +### What Could Improve +- FR-5 fallback refs are declared but not yet exercised at download time (github source would need retry logic) +- Could add integration tests against real APIs (currently all mocked) + +### Tech Debt Created +- `sources/npm.ts` marked deprecated but not removed — needs cleanup in a follow-up release +- FR-5 fallback ref retry logic not wired into github source fetch path + ## Dependencies - **Soft dependency** on `cli-shorthand-20260407`: once the github fast-path lands, the resolver can reuse the same code path. Parallel work is fine, but landing `cli-shorthand` first is preferable. diff --git a/.please/docs/tracks/active/ecosystem-resolvers-20260407/spec.md b/.please/docs/tracks/completed/ecosystem-resolvers-20260407/spec.md similarity index 100% rename from .please/docs/tracks/active/ecosystem-resolvers-20260407/spec.md rename to .please/docs/tracks/completed/ecosystem-resolvers-20260407/spec.md diff --git a/CLAUDE.md b/CLAUDE.md index 7edac35..3bd35e4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,8 @@ node packages/cli/dist/index.js docs add -s [options] - A PreToolUse Bash hook blocks shell commands when the current commit has no recorded review. Run `/review:code-review` (or `/review:run-cubic` / `/review:run-gemini`) and let it call `save-review-state.sh` before further bash. - `apps/registry/` uses `vercel.ts` (programmatic config via `@vercel/config` devDep), not `vercel.json`. Use `git.deploymentEnabled` (not deprecated `github.enabled`) to control auto-deploy. - Track artifacts live under `.please/docs/tracks/active/{slug}-{YYYYMMDD}/` with `spec.md`, `plan.md`, `metadata.json`. Append a JSON line to `.please/docs/tracks.jsonl` when creating a track. +- In git worktrees, `bun install` must be run before tests — dependencies are not shared across worktrees. +- `style/quote-props` ESLint rule: if any property in an interface requires quotes (e.g. `'dist-tags'`), all properties must be quoted for consistency. ## CLI Architecture (packages/cli/) @@ -65,12 +67,19 @@ node packages/cli/dist/index.js docs add -s [options] **Source adapter pattern** (`src/sources/`): - `index.ts` — defines `DocSource` interface, `SourceConfig` union type, and `getSource()` factory -- `npm.ts` — downloads npm tarballs, extracts doc files from package +- `npm.ts` — downloads npm tarballs, extracts doc files from package (deprecated — use resolver + github source) - `github.ts` — downloads GitHub repo archives via tar.gz, extracts docs directory - `web.ts` — crawls documentation websites, converts HTML to Markdown via `node-html-markdown` All three sources implement `DocSource.fetch(options) -> Promise` returning `{ files: DocFile[], resolvedVersion: string }`. +**Ecosystem resolver pattern** (`src/resolvers/`): +- `index.ts` — defines `EcosystemResolver` interface, `ResolveResult` type, and `getResolver()` factory +- `npm.ts` / `pypi.ts` / `pub.ts` — fetch package metadata APIs, extract GitHub `owner/repo`, delegate to github source +- `utils.ts` — `parseRepoUrl(url)` normalizes varied GitHub URL formats to `owner/repo` + +Resolvers are orthogonal to sources — they only perform metadata lookups and hand off `repo` + `ref` to the github source. The `add` command tries the registry first; resolvers activate on registry miss for ecosystem-prefixed specs. + **Output pipeline** (executed in sequence by `add` command): 1. `storage.ts` — saves doc files to `.ask/docs/@/`, creates `INDEX.md` 2. `config.ts` — persists source config to `.ask/config.json` for `sync` re-download diff --git a/README.md b/README.md index 4e458c9..4681fa9 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,26 @@ Description and version notes here... Add a new `.md` file under `apps/registry/content/registry//` and submit a PR. +## Ecosystem Resolvers + +When a library isn't in the registry, ASK can **automatically resolve** its GitHub repository from ecosystem package metadata. This works for any ecosystem-prefixed spec: + +```bash +ask docs add npm:lodash # Looks up registry.npmjs.org → lodash/lodash +ask docs add pypi:fastapi # Looks up pypi.org → fastapi/fastapi +ask docs add pub:riverpod # Looks up pub.dev → rrousselGit/riverpod +``` + +**Supported ecosystems:** + +| Ecosystem | API | Metadata field | +|---|---|---| +| `npm` | `registry.npmjs.org/` | `repository.url` | +| `pypi` | `pypi.org/pypi//json` | `info.project_urls.Source` | +| `pub` | `pub.dev/api/packages/` | `latest.pubspec.repository` | + +The resolver extracts the GitHub `owner/repo` from the package metadata and delegates to the GitHub source for download. The registry is always checked first — resolvers only activate on a registry miss. + ## Source Adapters ### npm diff --git a/bun.lock b/bun.lock index d6f411d..23c5643 100644 --- a/bun.lock +++ b/bun.lock @@ -37,11 +37,13 @@ "citty": "^0.2.2", "consola": "^3.4.2", "node-html-markdown": "^1.3.0", + "semver": "^7.7.4", "zod": "^3.23.0", }, "devDependencies": { "@pleaseai/eslint-config": "^0.0.1", "@types/node": "^22.0.0", + "@types/semver": "^7.7.1", "eslint": "^10.1.0", "jiti": "^2.6.1", "typescript": "^5.7.0", @@ -744,6 +746,8 @@ "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="], + "@types/semver": ["@types/semver@7.7.1", "", {}, "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], diff --git a/packages/cli/package.json b/packages/cli/package.json index 796f0b2..8484a59 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,11 +26,13 @@ "citty": "^0.2.2", "consola": "^3.4.2", "node-html-markdown": "^1.3.0", + "semver": "^7.7.4", "zod": "^3.23.0" }, "devDependencies": { "@pleaseai/eslint-config": "^0.0.1", "@types/node": "^22.0.0", + "@types/semver": "^7.7.1", "eslint": "^10.1.0", "jiti": "^2.6.1", "typescript": "^5.7.0" diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index cb19c99..e8fe32c 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -24,6 +24,7 @@ import { import { contentHash, getConfigPath, getLockPath, readLock, upsertLockEntry } from './io.js' import { migrateLegacyWorkspace } from './migrate-legacy.js' import { parseDocSpec, parseEcosystem, resolveFromRegistry } from './registry.js' +import { getResolver } from './resolvers/index.js' import { generateSkill, removeSkill } from './skill.js' import { getSource } from './sources/index.js' import { listDocs, removeDocs, saveDocs } from './storage.js' @@ -202,22 +203,48 @@ const addCmd = defineCommand({ else { // Auto-detect from registry const resolved = await resolveFromRegistry(args.spec, projectDir) - if (!resolved) { + if (resolved) { + const { strategy } = resolved + consola.start(`Downloading ${resolved.name}@${resolved.version} docs (source: ${strategy.source})...`) + sourceConfig = buildSourceConfig(resolved.name, resolved.version, { + source: strategy.source, + repo: strategy.repo, + docsPath: strategy.docsPath, + url: strategy.urls ?? (strategy.url ? [strategy.url] : undefined), + maxDepth: strategy.maxDepth?.toString(), + pathPrefix: strategy.allowedPathPrefix, + branch: strategy.branch, + tag: strategy.tag, + }) + } + else if (parsed.kind === 'ecosystem') { + // Registry miss with ecosystem prefix → try ecosystem resolver + const resolver = getResolver(parsed.ecosystem) + if (!resolver) { + consola.error( + `'${args.spec}' not found in registry and no resolver for '${parsed.ecosystem}'. ` + + `Use --source to specify manually.`, + ) + process.exit(1) + return // unreachable — hints TS that control flow ends + } + consola.info(`Registry miss — resolving via ${parsed.ecosystem} package metadata...`) + const resolveResult = await resolver.resolve(parsed.name, parsed.version) + consola.start(`Downloading ${parsed.name}@${resolveResult.resolvedVersion} docs (source: github via ${parsed.ecosystem} resolver)...`) + sourceConfig = { + source: 'github', + name: parsed.name, + version: resolveResult.resolvedVersion, + repo: resolveResult.repo, + tag: resolveResult.ref, + docsPath: args.docsPath, + } satisfies GithubSourceOptions + } + else { consola.error(`'${args.spec}' not found in registry. Use --source to specify manually.`) process.exit(1) + return // unreachable — hints TS that control flow ends } - const { strategy } = resolved - consola.start(`Downloading ${resolved.name}@${resolved.version} docs (source: ${strategy.source})...`) - sourceConfig = buildSourceConfig(resolved.name, resolved.version, { - source: strategy.source, - repo: strategy.repo, - docsPath: strategy.docsPath, - url: strategy.urls ?? (strategy.url ? [strategy.url] : undefined), - maxDepth: strategy.maxDepth?.toString(), - pathPrefix: strategy.allowedPathPrefix, - branch: strategy.branch, - tag: strategy.tag, - }) } const source = getSource(sourceConfig.source) diff --git a/packages/cli/src/resolvers/index.ts b/packages/cli/src/resolvers/index.ts new file mode 100644 index 0000000..032a879 --- /dev/null +++ b/packages/cli/src/resolvers/index.ts @@ -0,0 +1,47 @@ +import { NpmResolver } from './npm.js' +import { PubResolver } from './pub.js' +import { PypiResolver } from './pypi.js' + +/** + * Result of resolving an ecosystem package to a GitHub repository. + * Always handed off to the github source for download. + */ +export interface ResolveResult { + /** GitHub `owner/repo` */ + repo: string + /** Primary git ref to try first (tag or branch) */ + ref: string + /** + * Fallback refs to try if the primary ref doesn't exist. + * Implements FR-5: try `v{version}`, then `{version}`, then default branch. + */ + fallbackRefs?: string[] + /** Resolved version string (e.g. `4.17.21`) */ + resolvedVersion: string +} + +/** + * Ecosystem resolver: maps a package name + version to a GitHub repository. + * + * Resolvers are orthogonal to sources — they only perform metadata lookups + * and never download documentation themselves. The resolved `repo` + `ref` + * are handed to the `github` source for the actual download. + */ +export interface EcosystemResolver { + resolve: (name: string, version: string) => Promise +} + +type SupportedEcosystem = 'npm' | 'pypi' | 'pub' + +const resolvers: Record = { + npm: new NpmResolver(), + pypi: new PypiResolver(), + pub: new PubResolver(), +} + +/** + * Return the resolver for the given ecosystem, or `null` if unsupported. + */ +export function getResolver(ecosystem: string): EcosystemResolver | null { + return (resolvers as Record)[ecosystem] ?? null +} diff --git a/packages/cli/src/resolvers/npm.ts b/packages/cli/src/resolvers/npm.ts new file mode 100644 index 0000000..a12a8c0 --- /dev/null +++ b/packages/cli/src/resolvers/npm.ts @@ -0,0 +1,92 @@ +import type { EcosystemResolver, ResolveResult } from './index.js' +import { consola } from 'consola' +import { maxSatisfying, validRange } from 'semver' +import { parseRepoUrl } from './utils.js' + +const RE_SEMVER_RANGE_CHARS = /[~^>=<|]/g + +/** + * npm registry API response (partial — only fields we need). + */ +interface NpmPackageMeta { + 'repository'?: { type?: string, url?: string } | string + 'dist-tags'?: Record + 'versions'?: Record +} + +/** + * Resolve an npm package to a GitHub repo + git tag. + * + * 1. Fetch `https://registry.npmjs.org/` + * 2. Resolve version: dist-tag → exact, semver range → best match, exact → passthrough + * 3. Extract `repository.url` → `owner/repo` + * 4. Return `v{version}` as the git ref + */ +export class NpmResolver implements EcosystemResolver { + async resolve(name: string, version: string): Promise { + const url = `https://registry.npmjs.org/${name}` + const response = await fetch(url) + if (!response.ok) { + throw new Error(`npm registry returned ${response.status} for ${name}`) + } + + const meta = await response.json() as NpmPackageMeta + + const distTags = meta['dist-tags'] ?? {} + const allVersions = meta.versions ? Object.keys(meta.versions) : [] + + // Resolve version: dist-tag → semver range → exact + let resolvedVersion: string + if (distTags[version]) { + // dist-tag (e.g. 'latest', 'canary') + resolvedVersion = distTags[version] + } + else if (validRange(version) && version !== version.replace(RE_SEMVER_RANGE_CHARS, '')) { + // Semver range (e.g. '^15', '~3.22', '>=18.0.0') + const best = maxSatisfying(allVersions, version) + if (!best) { + throw new Error( + `No version matching '${version}' found for npm package '${name}'. ` + + `Available dist-tags: ${Object.keys(distTags).join(', ')}`, + ) + } + resolvedVersion = best + } + else { + // Exact version string + resolvedVersion = version + } + + // Verify the resolved version exists + if (allVersions.length > 0 && !allVersions.includes(resolvedVersion)) { + throw new Error( + `Version '${resolvedVersion}' not found for npm package '${name}'. ` + + `Available dist-tags: ${Object.keys(distTags).join(', ')}`, + ) + } + + // Extract repository URL + const repoField = meta.repository + const repoUrl = typeof repoField === 'string' + ? repoField + : repoField?.url + + const repo = parseRepoUrl(repoUrl) + if (!repo) { + throw new Error( + `Cannot resolve GitHub repository for npm package '${name}'. ` + + `The 'repository' field is missing or not a GitHub URL. ` + + `Use 'owner/repo' format instead: ask docs add owner/repo`, + ) + } + + consola.debug(`npm: ${name}@${version} → ${repo}@${resolvedVersion}`) + + return { + repo, + ref: `v${resolvedVersion}`, + fallbackRefs: [resolvedVersion], + resolvedVersion, + } + } +} diff --git a/packages/cli/src/resolvers/pub.ts b/packages/cli/src/resolvers/pub.ts new file mode 100644 index 0000000..f782949 --- /dev/null +++ b/packages/cli/src/resolvers/pub.ts @@ -0,0 +1,74 @@ +import type { EcosystemResolver, ResolveResult } from './index.js' +import { consola } from 'consola' +import { parseRepoUrl } from './utils.js' + +/** + * pub.dev API response (partial — only fields we need). + */ +interface PubPackageMeta { + latest: { + version: string + pubspec: { + repository?: string + homepage?: string + } + } + versions?: Array<{ version: string }> +} + +/** + * Resolve a pub.dev package to a GitHub repo + git tag. + * + * 1. Fetch `https://pub.dev/api/packages/` + * 2. If version is explicit, verify it exists + * 3. Extract `latest.pubspec.repository` → `owner/repo` + * 4. Use the version as the git ref + */ +export class PubResolver implements EcosystemResolver { + async resolve(name: string, version: string): Promise { + const url = `https://pub.dev/api/packages/${name}` + const response = await fetch(url) + if (!response.ok) { + throw new Error(`pub.dev returned ${response.status} for ${name}`) + } + + const meta = await response.json() as PubPackageMeta + const latestVersion = meta.latest.version + + let resolvedVersion: string + if (version === 'latest') { + resolvedVersion = latestVersion + } + else { + // Verify the requested version exists + const allVersions = meta.versions?.map(v => v.version) ?? [] + if (allVersions.length > 0 && !allVersions.includes(version)) { + throw new Error( + `Version '${version}' not found for pub package '${name}'. ` + + `Latest version: ${latestVersion}`, + ) + } + resolvedVersion = version + } + + // Extract repository URL + const repoUrl = meta.latest.pubspec.repository ?? meta.latest.pubspec.homepage + const repo = parseRepoUrl(repoUrl) + if (!repo) { + throw new Error( + `Cannot resolve GitHub repository for pub package '${name}'. ` + + `The 'repository' field is missing or not a GitHub URL. ` + + `Use 'owner/repo' format instead: ask docs add owner/repo`, + ) + } + + consola.debug(`pub: ${name}@${version} → ${repo}@${resolvedVersion}`) + + return { + repo, + ref: resolvedVersion, + fallbackRefs: [`v${resolvedVersion}`], + resolvedVersion, + } + } +} diff --git a/packages/cli/src/resolvers/pypi.ts b/packages/cli/src/resolvers/pypi.ts new file mode 100644 index 0000000..44a6260 --- /dev/null +++ b/packages/cli/src/resolvers/pypi.ts @@ -0,0 +1,83 @@ +import type { EcosystemResolver, ResolveResult } from './index.js' +import { consola } from 'consola' +import { parseRepoUrl } from './utils.js' + +/** + * PyPI JSON API response (partial — only fields we need). + */ +interface PypiPackageMeta { + info: { + version: string + project_urls?: Record | null + home_page?: string | null + } +} + +/** Keys under `project_urls` most likely to contain a source-code link. */ +const SOURCE_URL_KEYS = [ + 'Source', + 'Source Code', + 'Repository', + 'GitHub', + 'Code', + 'Homepage', +] + +/** + * Resolve a PyPI package to a GitHub repo + git tag. + * + * 1. Fetch `https://pypi.org/pypi//json` (or `//json`) + * 2. Extract `info.project_urls` → find a GitHub URL + * 3. Use the PyPI version as the git ref + */ +export class PypiResolver implements EcosystemResolver { + async resolve(name: string, version: string): Promise { + const isExplicit = version !== 'latest' + const url = isExplicit + ? `https://pypi.org/pypi/${name}/${version}/json` + : `https://pypi.org/pypi/${name}/json` + + const response = await fetch(url) + if (!response.ok) { + throw new Error(`PyPI returned ${response.status} for ${name}${isExplicit ? `@${version}` : ''}`) + } + + const meta = await response.json() as PypiPackageMeta + const resolvedVersion = meta.info.version + + // Find GitHub URL from project_urls + const projectUrls = meta.info.project_urls ?? {} + let repoUrl: string | null = null + + for (const key of SOURCE_URL_KEYS) { + const candidate = projectUrls[key] + if (candidate && candidate.includes('github.com')) { + repoUrl = candidate + break + } + } + + // Fall back to home_page + if (!repoUrl && meta.info.home_page?.includes('github.com')) { + repoUrl = meta.info.home_page + } + + const repo = parseRepoUrl(repoUrl) + if (!repo) { + throw new Error( + `Cannot resolve GitHub repository for PyPI package '${name}'. ` + + `The 'project_urls' field does not contain a GitHub URL. ` + + `Use 'owner/repo' format instead: ask docs add owner/repo`, + ) + } + + consola.debug(`pypi: ${name}@${version} → ${repo}@${resolvedVersion}`) + + return { + repo, + ref: `v${resolvedVersion}`, + fallbackRefs: [resolvedVersion], + resolvedVersion, + } + } +} diff --git a/packages/cli/src/resolvers/utils.ts b/packages/cli/src/resolvers/utils.ts new file mode 100644 index 0000000..f4ced86 --- /dev/null +++ b/packages/cli/src/resolvers/utils.ts @@ -0,0 +1,28 @@ +const RE_GITHUB_URL = /github\.com[/:]([^/]+)\/([^/#?\s]+)/ +const RE_DOT_GIT = /\.git$/ + +/** + * Parse a repository URL into `owner/repo` form. + * + * Handles common formats: + * - `git+https://github.com/owner/repo.git` + * - `https://github.com/owner/repo` + * - `git://github.com/owner/repo.git` + * - `ssh://git@github.com/owner/repo.git` + * - `github.com/owner/repo` + * - URLs with extra path segments (`/tree/main`, etc.) + * + * Returns `null` for non-GitHub URLs, empty strings, or undefined. + */ +export function parseRepoUrl(url: string | undefined | null): string | null { + if (!url) + return null + + const match = RE_GITHUB_URL.exec(url) + if (!match) + return null + + const owner = match[1] + const repo = match[2].replace(RE_DOT_GIT, '') + return `${owner}/${repo}` +} diff --git a/packages/cli/src/sources/npm.ts b/packages/cli/src/sources/npm.ts index 504ebf1..6bdd19f 100644 --- a/packages/cli/src/sources/npm.ts +++ b/packages/cli/src/sources/npm.ts @@ -11,6 +11,11 @@ import os from 'node:os' import path from 'node:path' import { consola } from 'consola' +/** + * @deprecated Use the npm ecosystem resolver (`resolvers/npm.ts`) + github source + * instead. This direct npm tarball source is kept for one release to avoid + * regressions in existing configs that specify `source: 'npm'` explicitly. + */ export class NpmSource implements DocSource { async fetch(options: SourceConfig): Promise { const opts = options as NpmSourceOptions diff --git a/packages/cli/test/resolvers/index.test.ts b/packages/cli/test/resolvers/index.test.ts new file mode 100644 index 0000000..61e4c7f --- /dev/null +++ b/packages/cli/test/resolvers/index.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'bun:test' +import { getResolver } from '../../src/resolvers/index.js' + +describe('getResolver', () => { + it('returns a resolver for npm', () => { + const resolver = getResolver('npm') + expect(resolver).toBeDefined() + expect(typeof resolver.resolve).toBe('function') + }) + + it('returns a resolver for pypi', () => { + const resolver = getResolver('pypi') + expect(resolver).toBeDefined() + expect(typeof resolver.resolve).toBe('function') + }) + + it('returns a resolver for pub', () => { + const resolver = getResolver('pub') + expect(resolver).toBeDefined() + expect(typeof resolver.resolve).toBe('function') + }) + + it('returns null for unsupported ecosystem', () => { + expect(getResolver('cargo')).toBeNull() + }) +}) diff --git a/packages/cli/test/resolvers/npm.test.ts b/packages/cli/test/resolvers/npm.test.ts new file mode 100644 index 0000000..6fc8534 --- /dev/null +++ b/packages/cli/test/resolvers/npm.test.ts @@ -0,0 +1,132 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test' +import { NpmResolver } from '../../src/resolvers/npm.js' + +describe('NpmResolver', () => { + const resolver = new NpmResolver() + const originalFetch = globalThis.fetch + + beforeEach(() => { + // Mock global fetch + globalThis.fetch = mock() as typeof fetch + }) + + afterEach(() => { + globalThis.fetch = originalFetch + }) + + function mockFetchJson(data: unknown, status = 200) { + ;(globalThis.fetch as ReturnType).mockResolvedValue( + new Response(JSON.stringify(data), { status }), + ) + } + + it('resolves a package with dist-tag "latest"', async () => { + mockFetchJson({ + 'repository': { type: 'git', url: 'git+https://github.com/lodash/lodash.git' }, + 'dist-tags': { latest: '4.17.21' }, + 'versions': { '4.17.21': {} }, + }) + + const result = await resolver.resolve('lodash', 'latest') + expect(result).toEqual({ + repo: 'lodash/lodash', + ref: 'v4.17.21', + fallbackRefs: ['4.17.21'], + resolvedVersion: '4.17.21', + }) + }) + + it('resolves an explicit version', async () => { + mockFetchJson({ + 'repository': { url: 'https://github.com/colinhacks/zod.git' }, + 'dist-tags': { latest: '3.23.8' }, + 'versions': { '3.22.4': {}, '3.23.8': {} }, + }) + + const result = await resolver.resolve('zod', '3.22.4') + expect(result).toEqual({ + repo: 'colinhacks/zod', + ref: 'v3.22.4', + fallbackRefs: ['3.22.4'], + resolvedVersion: '3.22.4', + }) + }) + + it('resolves a string-form repository field', async () => { + mockFetchJson({ + 'repository': 'https://github.com/expressjs/express', + 'dist-tags': { latest: '5.0.0' }, + 'versions': { '5.0.0': {} }, + }) + + const result = await resolver.resolve('express', 'latest') + expect(result.repo).toBe('expressjs/express') + }) + + it('throws when npm returns 404', async () => { + mockFetchJson({}, 404) + await expect(resolver.resolve('nonexistent', 'latest')).rejects.toThrow(/404/) + }) + + it('throws when repository field is missing', async () => { + mockFetchJson({ + 'dist-tags': { latest: '1.0.0' }, + 'versions': { '1.0.0': {} }, + }) + await expect(resolver.resolve('no-repo', 'latest')).rejects.toThrow(/Cannot resolve GitHub/) + }) + + it('throws when repository is not a GitHub URL', async () => { + mockFetchJson({ + 'repository': { url: 'https://gitlab.com/owner/repo.git' }, + 'dist-tags': { latest: '1.0.0' }, + 'versions': { '1.0.0': {} }, + }) + await expect(resolver.resolve('gitlab-pkg', 'latest')).rejects.toThrow(/Cannot resolve GitHub/) + }) + + it('throws when version not found in versions object', async () => { + mockFetchJson({ + 'repository': { url: 'https://github.com/owner/repo.git' }, + 'dist-tags': { latest: '1.0.0' }, + 'versions': { '1.0.0': {} }, + }) + await expect(resolver.resolve('pkg', '99.99.99')).rejects.toThrow(/not found/) + }) + + it('resolves a semver range like ^15', async () => { + mockFetchJson({ + 'repository': { url: 'https://github.com/vercel/next.js.git' }, + 'dist-tags': { latest: '15.2.3' }, + 'versions': { '14.2.0': {}, '15.0.0': {}, '15.1.0': {}, '15.2.3': {} }, + }) + + const result = await resolver.resolve('next', '^15') + expect(result).toEqual({ + repo: 'vercel/next.js', + ref: 'v15.2.3', + fallbackRefs: ['15.2.3'], + resolvedVersion: '15.2.3', + }) + }) + + it('resolves ~3.22 to highest 3.22.x', async () => { + mockFetchJson({ + 'repository': { url: 'https://github.com/colinhacks/zod.git' }, + 'dist-tags': { latest: '3.23.8' }, + 'versions': { '3.22.0': {}, '3.22.4': {}, '3.23.0': {}, '3.23.8': {} }, + }) + + const result = await resolver.resolve('zod', '~3.22') + expect(result.resolvedVersion).toBe('3.22.4') + }) + + it('throws when no version matches the semver range', async () => { + mockFetchJson({ + 'repository': { url: 'https://github.com/owner/repo.git' }, + 'dist-tags': { latest: '1.0.0' }, + 'versions': { '1.0.0': {} }, + }) + await expect(resolver.resolve('pkg', '^99')).rejects.toThrow(/No version matching/) + }) +}) diff --git a/packages/cli/test/resolvers/pub.test.ts b/packages/cli/test/resolvers/pub.test.ts new file mode 100644 index 0000000..ece6b44 --- /dev/null +++ b/packages/cli/test/resolvers/pub.test.ts @@ -0,0 +1,103 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test' +import { PubResolver } from '../../src/resolvers/pub.js' + +describe('PubResolver', () => { + const resolver = new PubResolver() + const originalFetch = globalThis.fetch + + beforeEach(() => { + globalThis.fetch = mock() as typeof fetch + }) + + afterEach(() => { + globalThis.fetch = originalFetch + }) + + function mockFetchJson(data: unknown, status = 200) { + ;(globalThis.fetch as ReturnType).mockResolvedValue( + new Response(JSON.stringify(data), { status }), + ) + } + + it('resolves a package with pubspec.repository', async () => { + mockFetchJson({ + latest: { + version: '2.6.1', + pubspec: { + repository: 'https://github.com/rrousselGit/riverpod', + }, + }, + versions: [{ version: '2.6.1' }, { version: '2.5.0' }], + }) + + const result = await resolver.resolve('riverpod', 'latest') + expect(result).toEqual({ + repo: 'rrousselGit/riverpod', + ref: '2.6.1', + fallbackRefs: ['v2.6.1'], + resolvedVersion: '2.6.1', + }) + }) + + it('resolves an explicit version', async () => { + mockFetchJson({ + latest: { + version: '2.6.1', + pubspec: { + repository: 'https://github.com/rrousselGit/riverpod', + }, + }, + versions: [{ version: '2.6.1' }, { version: '2.5.0' }], + }) + + const result = await resolver.resolve('riverpod', '2.5.0') + expect(result).toEqual({ + repo: 'rrousselGit/riverpod', + ref: '2.5.0', + fallbackRefs: ['v2.5.0'], + resolvedVersion: '2.5.0', + }) + }) + + it('falls back to homepage when repository is missing', async () => { + mockFetchJson({ + latest: { + version: '1.0.0', + pubspec: { + homepage: 'https://github.com/owner/repo', + }, + }, + }) + + const result = await resolver.resolve('pkg', 'latest') + expect(result.repo).toBe('owner/repo') + }) + + it('throws when pub.dev returns 404', async () => { + mockFetchJson({}, 404) + await expect(resolver.resolve('nonexistent', 'latest')).rejects.toThrow(/404/) + }) + + it('throws when no GitHub URL found', async () => { + mockFetchJson({ + latest: { + version: '1.0.0', + pubspec: {}, + }, + }) + await expect(resolver.resolve('no-repo', 'latest')).rejects.toThrow(/Cannot resolve GitHub/) + }) + + it('throws when requested version does not exist', async () => { + mockFetchJson({ + latest: { + version: '1.0.0', + pubspec: { + repository: 'https://github.com/owner/repo', + }, + }, + versions: [{ version: '1.0.0' }], + }) + await expect(resolver.resolve('pkg', '99.0.0')).rejects.toThrow(/not found/) + }) +}) diff --git a/packages/cli/test/resolvers/pypi.test.ts b/packages/cli/test/resolvers/pypi.test.ts new file mode 100644 index 0000000..5a05fad --- /dev/null +++ b/packages/cli/test/resolvers/pypi.test.ts @@ -0,0 +1,116 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test' +import { PypiResolver } from '../../src/resolvers/pypi.js' + +describe('PypiResolver', () => { + const resolver = new PypiResolver() + const originalFetch = globalThis.fetch + + beforeEach(() => { + globalThis.fetch = mock() as typeof fetch + }) + + afterEach(() => { + globalThis.fetch = originalFetch + }) + + function mockFetchJson(data: unknown, status = 200) { + ;(globalThis.fetch as ReturnType).mockResolvedValue( + new Response(JSON.stringify(data), { status }), + ) + } + + it('resolves a package with project_urls.Source', async () => { + mockFetchJson({ + info: { + version: '0.115.0', + project_urls: { + Source: 'https://github.com/fastapi/fastapi', + Documentation: 'https://fastapi.tiangolo.com/', + }, + }, + }) + + const result = await resolver.resolve('fastapi', 'latest') + expect(result).toEqual({ + repo: 'fastapi/fastapi', + ref: 'v0.115.0', + fallbackRefs: ['0.115.0'], + resolvedVersion: '0.115.0', + }) + }) + + it('resolves explicit version via /pypi///json', async () => { + mockFetchJson({ + info: { + version: '0.110.0', + project_urls: { + 'Source Code': 'https://github.com/fastapi/fastapi', + }, + }, + }) + + const result = await resolver.resolve('fastapi', '0.110.0') + expect(result.resolvedVersion).toBe('0.110.0') + expect(globalThis.fetch).toHaveBeenCalledWith( + 'https://pypi.org/pypi/fastapi/0.110.0/json', + ) + }) + + it('finds repo via Repository key', async () => { + mockFetchJson({ + info: { + version: '2.0.0', + project_urls: { + Repository: 'https://github.com/pallets/flask', + }, + }, + }) + + const result = await resolver.resolve('flask', 'latest') + expect(result.repo).toBe('pallets/flask') + }) + + it('falls back to home_page when project_urls has no GitHub link', async () => { + mockFetchJson({ + info: { + version: '1.0.0', + project_urls: { + Documentation: 'https://docs.example.com', + }, + home_page: 'https://github.com/owner/repo', + }, + }) + + const result = await resolver.resolve('pkg', 'latest') + expect(result.repo).toBe('owner/repo') + }) + + it('throws when PyPI returns 404', async () => { + mockFetchJson({}, 404) + await expect(resolver.resolve('nonexistent', 'latest')).rejects.toThrow(/404/) + }) + + it('throws when no GitHub URL found', async () => { + mockFetchJson({ + info: { + version: '1.0.0', + project_urls: { + Documentation: 'https://docs.example.com', + }, + home_page: 'https://example.com', + }, + }) + await expect(resolver.resolve('no-gh', 'latest')).rejects.toThrow(/Cannot resolve GitHub/) + }) + + it('throws when project_urls is null', async () => { + mockFetchJson({ + info: { + version: '1.0.0', + project_urls: null, + home_page: null, + }, + }) + await expect(resolver.resolve('empty', 'latest')).rejects.toThrow(/Cannot resolve GitHub/) + }) +}) diff --git a/packages/cli/test/resolvers/regression.test.ts b/packages/cli/test/resolvers/regression.test.ts new file mode 100644 index 0000000..8bd2125 --- /dev/null +++ b/packages/cli/test/resolvers/regression.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'bun:test' +import { parseDocSpec } from '../../src/registry.js' +import { getResolver } from '../../src/resolvers/index.js' + +describe('resolver regression', () => { + it('registry-hit specs do not trigger resolvers', () => { + // Bare names parse as 'name' kind, not 'ecosystem' kind, + // so the add command's resolver fallback branch is never reached. + const bare = parseDocSpec('next') + expect(bare.kind).toBe('name') + + // Even with an ecosystem prefix, the registry is checked first. + // The resolver is only used when the registry returns null. + const prefixed = parseDocSpec('npm:next') + expect(prefixed.kind).toBe('ecosystem') + }) + + it('getResolver returns null for unsupported ecosystems', () => { + expect(getResolver('cargo')).toBeNull() + expect(getResolver('go')).toBeNull() + expect(getResolver('hex')).toBeNull() + expect(getResolver('nuget')).toBeNull() + }) + + it('getResolver returns resolvers for supported ecosystems', () => { + expect(getResolver('npm')).not.toBeNull() + expect(getResolver('pypi')).not.toBeNull() + expect(getResolver('pub')).not.toBeNull() + }) +}) diff --git a/packages/cli/test/resolvers/utils.test.ts b/packages/cli/test/resolvers/utils.test.ts new file mode 100644 index 0000000..940ee4e --- /dev/null +++ b/packages/cli/test/resolvers/utils.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'bun:test' +import { parseRepoUrl } from '../../src/resolvers/utils.js' + +describe('parseRepoUrl', () => { + it('parses git+https://github.com/owner/repo.git', () => { + expect(parseRepoUrl('git+https://github.com/owner/repo.git')).toBe('owner/repo') + }) + + it('parses https://github.com/owner/repo.git', () => { + expect(parseRepoUrl('https://github.com/owner/repo.git')).toBe('owner/repo') + }) + + it('parses https://github.com/owner/repo', () => { + expect(parseRepoUrl('https://github.com/owner/repo')).toBe('owner/repo') + }) + + it('parses git://github.com/owner/repo.git', () => { + expect(parseRepoUrl('git://github.com/owner/repo.git')).toBe('owner/repo') + }) + + it('parses ssh://git@github.com/owner/repo.git', () => { + expect(parseRepoUrl('ssh://git@github.com/owner/repo.git')).toBe('owner/repo') + }) + + it('parses github.com/owner/repo (no protocol)', () => { + expect(parseRepoUrl('github.com/owner/repo')).toBe('owner/repo') + }) + + it('strips trailing .git', () => { + expect(parseRepoUrl('https://github.com/lodash/lodash.git')).toBe('lodash/lodash') + }) + + it('strips trailing slashes', () => { + expect(parseRepoUrl('https://github.com/owner/repo/')).toBe('owner/repo') + }) + + it('handles paths with extra segments (tree/main)', () => { + expect(parseRepoUrl('https://github.com/owner/repo/tree/main')).toBe('owner/repo') + }) + + it('returns null for non-github URLs', () => { + expect(parseRepoUrl('https://gitlab.com/owner/repo')).toBeNull() + }) + + it('returns null for empty string', () => { + expect(parseRepoUrl('')).toBeNull() + }) + + it('returns null for undefined', () => { + expect(parseRepoUrl(undefined)).toBeNull() + }) + + it('returns null for malformed URLs', () => { + expect(parseRepoUrl('not a url')).toBeNull() + }) +})