diff --git a/.github/instructions/oss-third-party-notices.instructions.md b/.github/instructions/oss-third-party-notices.instructions.md index 420e42f28bae04..69a4e631a585de 100644 --- a/.github/instructions/oss-third-party-notices.instructions.md +++ b/.github/instructions/oss-third-party-notices.instructions.md @@ -224,7 +224,7 @@ Section 4 closes two Rust gaps: Important functions and constants: - `parseCargoLock()` extracts `name`, `version`, and `source` from `[[package]]` blocks. -- `fetchCratesIoJson()` calls `https://crates.io/api/v1/crates/` with `CRATES_IO_USER_AGENT`. +- `fetchCratesIoJson()` calls the cargo registry metadata API with `CRATES_IO_USER_AGENT` (local/offline uses crates.io; CI prefers the Monaco feed registry endpoint). - `getCrateRepository()` applies legacy repository URL overrides for crates such as `isatty`, `redox_syscall`, `redox_termios`, and `termion`. - `crateLicenseRefs()` tries `v`, ``, `-v`, `-`, then `main` and `master`. - `isSpdxStub()` detects CG bodies that are only SPDX expressions. diff --git a/build/azure-pipelines/alpine/product-build-alpine-node-modules.yml b/build/azure-pipelines/alpine/product-build-alpine-node-modules.yml index f819c45e99fcf8..991e1527331076 100644 --- a/build/azure-pipelines/alpine/product-build-alpine-node-modules.yml +++ b/build/azure-pipelines/alpine/product-build-alpine-node-modules.yml @@ -52,13 +52,13 @@ jobs: # which does not have their own .npmrc file npm config set registry "$NPM_REGISTRY" echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)" - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM - task: npmAuthenticate@0 inputs: workingFile: $(NPMRC_PATH) - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM Authentication - task: Docker@1 diff --git a/build/azure-pipelines/darwin/product-build-darwin-node-modules.yml b/build/azure-pipelines/darwin/product-build-darwin-node-modules.yml index 221a23bda89f21..fa516b14d69826 100644 --- a/build/azure-pipelines/darwin/product-build-darwin-node-modules.yml +++ b/build/azure-pipelines/darwin/product-build-darwin-node-modules.yml @@ -60,19 +60,18 @@ jobs: # which does not have their own .npmrc file npm config set registry "$NPM_REGISTRY" echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)" - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM - task: npmAuthenticate@0 inputs: workingFile: $(NPMRC_PATH) - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM Authentication - task: PipAuthenticate@1 inputs: artifactFeeds: Monaco/vscode - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) displayName: Setup PyPI Authentication - script: | diff --git a/build/azure-pipelines/linux/product-build-linux-node-modules.yml b/build/azure-pipelines/linux/product-build-linux-node-modules.yml index 4e2ecb9e779608..525120ed8cb368 100644 --- a/build/azure-pipelines/linux/product-build-linux-node-modules.yml +++ b/build/azure-pipelines/linux/product-build-linux-node-modules.yml @@ -73,13 +73,13 @@ jobs: # which does not have their own .npmrc file npm config set registry "$NPM_REGISTRY" echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)" - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM - task: npmAuthenticate@0 inputs: workingFile: $(NPMRC_PATH) - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) + condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM Authentication - script: | diff --git a/build/azure-pipelines/oss/scan-licenses.ts b/build/azure-pipelines/oss/scan-licenses.ts index 9997a8ce31e554..afc9a789c150ce 100644 --- a/build/azure-pipelines/oss/scan-licenses.ts +++ b/build/azure-pipelines/oss/scan-licenses.ts @@ -9,7 +9,7 @@ * Section 1: Built-in extension dependencies (CG skips engines.vscode packages) * Section 2: Root node_modules ClearlyDefined gaps (LICENSE on disk but not in CG output) * Section 3: cgmanifest.json git components CG can't harvest from ClearlyDefined - * Section 4: Cargo.lock crates missing or stubbed in CG (crates.io + repo LICENSE) + * Section 4: Cargo.lock crates missing or stubbed in CG (cargo registry API + repo LICENSE) * Section 5: Platform-specific binary packages (arch optionalDependencies) * Section 6: OS-gated whole packages skipped by npm on the build host (lockfile) * Section 7: Pre-built built-in extension deps (disk extensionsCG//, else @@ -31,6 +31,9 @@ import { fetchUriText } from './apply-overrides.js'; import { parseNoticeFile } from './parse-notices.js'; import { parseArgs } from './utils.js'; +const VSCODE_CFS_NPM_REGISTRY = 'https://pkgs.dev.azure.com/monacotools/Monaco/_packaging/vscode/npm/registry/'; +const VSCODE_CFS_CARGO_API_BASE = 'https://pkgs.dev.azure.com/monacotools/Monaco/_packaging/vscode/Cargo/api/v1/crates/'; + interface LicenseEntry { name: string; version: string; @@ -52,6 +55,18 @@ const COPYRIGHT_PATTERN = /copyright|(\(c\)|©)/i; /** Minimum length for a license body to be considered real (not a symlink stub or SPDX stub). */ const MIN_LICENSE_BODY_LENGTH = 40; +function isCI(): boolean { + return !!(process.env.TF_BUILD || process.env.BUILD_BUILDID); +} + +function resolveNpmPackumentBaseUrl(): string { + return isCI() ? VSCODE_CFS_NPM_REGISTRY : 'https://registry.npmjs.org/'; +} + +function resolveCargoApiBaseUrl(): string { + return isCI() ? VSCODE_CFS_CARGO_API_BASE : 'https://crates.io/api/v1/crates/'; +} + function validateCopyright(name: string, licenseText: string, source: string): void { if (!COPYRIGHT_PATTERN.test(licenseText)) { // allow-any-unicode-next-line @@ -133,7 +148,7 @@ export async function fetchLicenseFromGitRepo(repositoryUrl: string, commitHash: // entry-point guard at the bottom of this file, mirroring parse-notices.ts). // ============================================================================= -/** The User-Agent crates.io requires -- it rejects requests without a descriptive one. */ +/** The crates metadata API requires a descriptive User-Agent. */ const CRATES_IO_USER_AGENT = 'vscode-oss-scanner'; export interface CrateInfo { @@ -191,10 +206,10 @@ export function parseCargoLock(content: string): CargoPackage[] { } /** - * Resolve a crate's canonical GitHub repo URL. Some crates.io `repository` + * Resolve a crate's canonical GitHub repo URL. Some registry `repository` * fields are wrong or point at non-GitHub mirrors; this ports the legacy * override map (distro-tools/lib/cargo.ts getRepository) verbatim and falls - * back to the crates.io-reported repository otherwise. + * back to the registry-reported repository otherwise. */ export function getCrateRepository(info: CrateInfo): string { switch (info.crate.id) { @@ -276,19 +291,17 @@ export function isSpdxStub(body: string): boolean { } /** - * GET https://crates.io/api/v1/crates/ with the required User-Agent. + * GET crates metadata with the required User-Agent. * fetchUriText() can't set headers, so we use a small dedicated fetcher here. * Resolves undefined on any failure (caller logs + continues -- never crashes). */ export function fetchCratesIoJson(name: string, timeoutMs = 10_000): Promise { return new Promise(resolve => { - const options: https.RequestOptions = { - host: 'crates.io', - path: `/api/v1/crates/${encodeURIComponent(name)}`, + const url = new URL(`${encodeURIComponent(name)}`, resolveCargoApiBaseUrl()); + const req = https.get(url, { headers: { 'User-Agent': CRATES_IO_USER_AGENT, 'Accept': 'application/json' }, timeout: timeoutMs, - }; - const req = https.get(options, res => { + }, res => { if (res.statusCode !== 200) { res.resume(); resolve(undefined); @@ -319,7 +332,7 @@ export function fetchCratesIoJson(name: string, timeoutMs = 10_000): Promise Promise>, limit: number): Promise { @@ -858,20 +871,19 @@ export interface NpmPackument { } /** - * Fetch a package's registry packument from registry.npmjs.org. Modeled on + * Fetch a package's registry packument. Modeled on * fetchCratesIoJson: resolves undefined on any non-200, parse error, network * error, or timeout so a registry hiccup never crashes the build. Exported for * testing (the unit tests do NOT call it over the network). */ export function fetchNpmRegistryJson(name: string, timeoutMs = 10_000): Promise { return new Promise(resolve => { - const options: https.RequestOptions = { - host: 'registry.npmjs.org', - path: '/' + encodeURIComponent(name), + const baseUrl = resolveNpmPackumentBaseUrl(); + const url = new URL(`${encodeURIComponent(name)}`, baseUrl); + const req = https.get(url, { headers: { 'Accept': 'application/json' }, timeout: timeoutMs, - }; - const req = https.get(options, res => { + }, res => { if (res.statusCode !== 200) { res.resume(); resolve(undefined); @@ -1388,7 +1400,7 @@ async function main(): Promise { // expression as the license BODY (e.g. "Zlib OR Apache-2.0 OR MIT") // instead of the real license text. // - // Both are closed by the same pipeline: parse Cargo.lock -> crates.io API -> + // Both are closed by the same pipeline: parse Cargo.lock -> cargo registry API -> // resolve repo URL -> fetch the REAL license text from the repo (tag-pinned). // CG-coverage gating bounds network calls to ~34 crates, not all ~419. // @@ -1484,7 +1496,7 @@ async function main(): Promise { } } - console.log(` Resolving ${pending.length} crates via crates.io (coverage-gated)`); + console.log(` Resolving ${pending.length} crates via cargo registry API (coverage-gated)`); const cargoTasks = pending.map(p => async (): Promise => { try { @@ -1500,7 +1512,7 @@ async function main(): Promise { // envelope `{errors:[…]}`, schema drift, missing `versions`/`crate`) // would make the dereferences below throw a TypeError, rejecting the // task → Promise.all → main() → process.exit(1), crashing the build. - // Spec sec. 6.6: a failed crates.io call must log and continue, never crash. + // Spec sec. 6.6: a failed cargo registry API call must log and continue, never crash. if (!Array.isArray(info.versions) || !info.crate || typeof info.crate.id !== 'string') { cargoApiFailed++; unresolved.push({ name: p.name, version: p.version, reason: 'cargo-api-failed' }); @@ -2179,7 +2191,7 @@ async function main(): Promise { console.log(` Skipped (CG covers w/ text): ${cargoSkippedCgCovered}`); console.log(` Added (coverage gap): ${cargoFetched}`); console.log(` Stub-overridden: ${cargoStubOverride}`); - console.log(` crates.io API failed: ${cargoApiFailed}`); + console.log(` Cargo registry API failed: ${cargoApiFailed}`); console.log(` Fetch failed (no LICENSE): ${cargoFetchFailed}`); console.log(` AND-license incomplete: ${cargoAndIncomplete}`); console.log(` Section 5 - Platform binaries:`); diff --git a/build/azure-pipelines/product-build.yml b/build/azure-pipelines/product-build.yml index 3be251349b301f..f587ae2dd1029e 100644 --- a/build/azure-pipelines/product-build.yml +++ b/build/azure-pipelines/product-build.yml @@ -283,7 +283,7 @@ extends: ubuntu-2004-arm64: image: onebranch.azurecr.io/linux/ubuntu-2004-arm64:latest settings: - networkIsolationPolicy: Permissive,CFSClean2,CFSClean3 + networkIsolationPolicy: Permissive,CFSClean,CFSClean2,CFSClean3 stages: - stage: Quality diff --git a/build/azure-pipelines/product-quality-checks.yml b/build/azure-pipelines/product-quality-checks.yml index 9c6f39afa0ae24..6710b2f70be0c8 100644 --- a/build/azure-pipelines/product-quality-checks.yml +++ b/build/azure-pipelines/product-quality-checks.yml @@ -55,6 +55,31 @@ jobs: condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM Authentication + - script: | + set -e + mkdir -p "$HOME/.cargo" + cat > "$HOME/.cargo/config.toml" <