diff --git a/.azure-pipelines/1ES.Build.Official.yml b/.azure-pipelines/1ES.Build.Official.yml index 1a3211d4d..8ab7d1dd1 100644 --- a/.azure-pipelines/1ES.Build.Official.yml +++ b/.azure-pipelines/1ES.Build.Official.yml @@ -28,6 +28,10 @@ parameters: displayName: Enable debug output for SDK tests type: boolean default: false + - name: packageCrates + displayName: Package crates.io release artifacts (blocked by Cargo #17196) + type: boolean + default: false variables: # Variable group managed in Dart — see the pipeline's variable group list. @@ -50,6 +54,7 @@ extends: - template: templates/1ES.Build.Stages.yml@self parameters: isOfficialBuild: true + packageCrates: ${{ parameters.packageCrates }} debug: ${{ parameters.debug }} ESRPInfo: serviceName: $(serviceName) diff --git a/.azure-pipelines/1ES.Release.yml b/.azure-pipelines/1ES.Release.yml index 6a86c5378..774a86d8f 100644 --- a/.azure-pipelines/1ES.Release.yml +++ b/.azure-pipelines/1ES.Release.yml @@ -1,5 +1,21 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# +# Single entry point for MXC package releases: one run can publish the npm SDK, +# the Rust crate closure to crates.io, or both, via the `publishNpm` / +# `publishCrates` parameters. Both publish through ESRP Release (no registry +# tokens in this repo), consuming artifacts from `MXC-Official-Build`: +# * npm SDK -> `mxc-npm-sdk-package` (Package.NpmSdk.Job.yml) +# * crates.io -> `mxc-crates-package` (Package.Crates.Job.yml) +# Leaf-first crate ordering/verification lives in templates/Publish.CratesIo.Job.yml. +# +# One-time enablement (out of band): OSPO OSS-release registration, crate-name +# reservation under `microsoft-oss-releases`, and ESRP `Rust` content-type +# onboarding for the ESRP client id (see the ESRP crates.io docs). The release +# pool's network-isolation policy must also allow read-only HTTPS access to +# `index.crates.io`; build-time dependency consumption remains on the private +# Mxc-Azure-Feed. No CARGO_REGISTRY_TOKEN or variable group is needed -- ESRP +# owns the credentials. trigger: none name: $(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) @@ -10,7 +26,7 @@ resources: name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release pipelines: - # Reference pipeline that created the signed npm artifacts so we can consume them later + # Reference pipeline that created the signed release artifacts so we can consume them later - pipeline: MXC source: 'MXC-Official-Build' trigger: none @@ -18,6 +34,43 @@ resources: # Parameters for ESRP release info will be passed from the ADO UI. parameters: + - name: publishNpm + displayName: "Publish the npm SDK to npmjs.com" + type: boolean + default: false + - name: publishCrates + displayName: "Publish the Rust crate closure to crates.io" + type: boolean + default: false + - name: cratesDryRun + displayName: "Crates dry run: stage + verify only, skip the ESRP publish" + type: boolean + default: false + # Leaf-first crate order for crates.io. MUST match the CRATES list in + # .azure-pipelines/scripts/crates_release.py; verified at run time by + # `verify-order`, which fails the release if the two disagree. + # + - name: crateOrder + type: object + default: + - nanvix_common + - mxc_telemetry + - wxc_common + - nanvix_runner + - hyperlight_common + - mxc_pty + - lxc_common + - bwrap_common + - seatbelt_common + - sandbox_spec + - appcontainer_common + - isolation_session_bindings + - isolation_session_common + - windows_sandbox_common + - windows_sandbox_lifecycle + - wslc_common + - mxc_engine + - mxc-sdk - name: ESRPInfo type: object default: @@ -47,42 +100,54 @@ extends: os: windows stages: - - stage: Publish_to_NPM - displayName: 'Publish NPM Package' - variables: - - name: release_environment - value: Production - jobs: - - job: ReleaseJob - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - pipeline: MXC - targetPath: '$(Pipeline.Workspace)/packages' - artifactName: mxc-npm-sdk-package + - ${{ if parameters.publishNpm }}: + - stage: Publish_to_NPM + displayName: 'Publish NPM Package' + variables: + - name: release_environment + value: Production + jobs: + - job: ReleaseJob + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: MXC + targetPath: '$(Pipeline.Workspace)/packages' + artifactName: mxc-npm-sdk-package + + displayName: Publish to NPM + steps: + - task: EsrpRelease@10 + displayName: 'Publish to NPM' + inputs: + connectedservicename: ${{ parameters.ESRPInfo.serviceName }} + usemanagedidentity: false + keyvaultname: ${{ parameters.ESRPInfo.azureKeyVaultName }} + authcertname: ${{ parameters.ESRPInfo.authCertName }} + signcertname: ${{ parameters.ESRPInfo.signCertName }} + clientid: ${{ parameters.ESRPInfo.clientId }} + intent: 'PackageDistribution' + contenttype: npm + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/packages' + owners: ${{ parameters.ESRPInfo.OwnersEmail }} + approvers: ${{ parameters.ESRPInfo.ApproversEmail }} + waitforreleasecompletion: true + serviceendpointurl: 'https://api.esrp.microsoft.com' + mainpublisher: ESRPRELPACMAN + domaintenantid: ${{ parameters.ESRPInfo.tenantId }} - displayName: Publish to NPM - steps: - - task: EsrpRelease@10 - displayName: 'Publish to NPM' - inputs: - connectedservicename: ${{ parameters.ESRPInfo.serviceName }} - usemanagedidentity: false - keyvaultname: ${{ parameters.ESRPInfo.azureKeyVaultName }} - authcertname: ${{ parameters.ESRPInfo.authCertName }} - signcertname: ${{ parameters.ESRPInfo.signCertName }} - clientid: ${{ parameters.ESRPInfo.clientId }} - intent: 'PackageDistribution' - contenttype: npm - contentsource: 'Folder' - folderlocation: '$(Pipeline.Workspace)/packages' - owners: ${{ parameters.ESRPInfo.OwnersEmail }} - approvers: ${{ parameters.ESRPInfo.ApproversEmail }} - waitforreleasecompletion: true - serviceendpointurl: 'https://api.esrp.microsoft.com' - mainpublisher: ESRPRELPACMAN - domaintenantid: ${{ parameters.ESRPInfo.tenantId }} + - ${{ if parameters.publishCrates }}: + - stage: Publish_to_CratesIo + displayName: 'Publish Crates.io Packages' + dependsOn: [] + jobs: + - template: templates/Publish.CratesIo.Job.yml@self + parameters: + ESRPInfo: ${{ parameters.ESRPInfo }} + crateOrder: ${{ parameters.crateOrder }} + cratesDryRun: ${{ parameters.cratesDryRun }} \ No newline at end of file diff --git a/.azure-pipelines/scripts/crates_release.py b/.azure-pipelines/scripts/crates_release.py new file mode 100644 index 000000000..1a2a31aa6 --- /dev/null +++ b/.azure-pipelines/scripts/crates_release.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +"""crates.io release helper for the `mxc-sdk` crate closure, ESRP edition. + +MXC publishes crates through ESRP Release under the official +`microsoft-oss-releases` account. ESRP accepts pre-built `.crate` files, so this +repository never handles a `CARGO_REGISTRY_TOKEN`. See: +https://eng.ms/docs/microsoft-security/identity/trust-and-security-services/tss-release-distribute/tss-release-esrp-parent/oss-publishing/releasing-open-source/cratesio + +ESRP does not sort a multi-crate dependency graph. The pipeline publishes one +crate at a time in leaf-first order. Before a dependent crate is submitted, this +helper confirms that every first-party dependency exists on the real crates.io +sparse index with the checksum produced by the same official build. + +Subcommands +----------- +package Validate and package the complete first-party closure, then write + `.crate` files and release-order.json. + +verify-order Assert that the pipeline's compile-time crate order exactly + matches the packaged order and its dependency graph. + +probe Confirm the release pool can read the crates.io sparse index. + +stage Copy one `.crate` file into a clean directory for an ESRP task. + +verify-dependencies + Confirm every first-party dependency of one packaged crate exists + on crates.io with the exact packaged checksum. + +status Set the pipeline's `crateAlreadyPublished` variable after checking + whether the exact packaged crate is already on crates.io. + +wait Poll crates.io until the exact packaged crate is visible before + the release continues to any dependent crate. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request + +# Current Cargo package names in leaf-first order. These names remain +# provisional until the public naming scheme is approved; publishCrates defaults +# to false in 1ES.Release.yml. +CRATES: list[str] = [ + "nanvix_common", + "mxc_telemetry", + "wxc_common", + "nanvix_runner", + "hyperlight_common", + "mxc_pty", + "lxc_common", + "bwrap_common", + "seatbelt_common", + "sandbox_spec", + "appcontainer_common", + "isolation_session_bindings", + "isolation_session_common", + "windows_sandbox_common", + "windows_sandbox_lifecycle", + "wslc_common", + "mxc_engine", + "mxc-sdk", +] + +CRATES_IO_SPARSE_INDEX = "https://index.crates.io" +PROPAGATION_TIMEOUT = 300 +PROPAGATION_POLL = 5 + + +def _sparse_index_path(name: str) -> str: + """Return the crates.io sparse-index path for a package name.""" + name = name.lower() + if len(name) == 1: + return f"1/{name}" + if len(name) == 2: + return f"2/{name}" + if len(name) == 3: + return f"3/{name[0]}/{name}" + return f"{name[:2]}/{name[2:4]}/{name}" + + +def _index_request(url: str) -> urllib.request.Request: + return urllib.request.Request( + url, + headers={"User-Agent": "mxc-crates-release/1.0"}, + ) + + +def _published_releases(crate: str) -> dict[str, dict]: + """Return crates.io sparse-index records for `crate`, keyed by version.""" + url = f"{CRATES_IO_SPARSE_INDEX}/{_sparse_index_path(crate)}" + try: + with urllib.request.urlopen(_index_request(url), timeout=30) as response: + body = response.read().decode("utf-8") + except urllib.error.HTTPError as error: + if error.code == 404: + return {} + raise + + releases: dict[str, dict] = {} + for line in body.splitlines(): + line = line.strip() + if line: + record = json.loads(line) + releases[record["vers"]] = record + return releases + + +def _cargo_metadata(manifest_path: str) -> dict: + result = subprocess.run( + [ + "cargo", + "metadata", + "--no-deps", + "--format-version", + "1", + "--manifest-path", + manifest_path, + ], + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout) + + +def _run(args: list[str], cwd: str | None = None) -> int: + print("+ " + " ".join(args), flush=True) + return subprocess.run(args, cwd=cwd).returncode + + +def _load_order(order_file: str) -> list[dict]: + with open(order_file, encoding="utf-8") as fh: + return json.load(fh)["crates"] + + +def _entry_for_crate(order_file: str, crate: str) -> dict | None: + return next( + ( + entry + for entry in _load_order(order_file) + if entry["name"] == crate + ), + None, + ) + + +def _crate_file(order_file: str, entry: dict) -> str: + return os.path.join( + os.path.dirname(os.path.abspath(order_file)), + entry["file"], + ) + + +def _sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _published_matches(order_file: str, entry: dict) -> bool: + """Return whether crates.io has this exact crate; reject conflicts.""" + record = _published_releases(entry["name"]).get(entry["version"]) + if record is None: + return False + if record.get("yanked", False): + raise RuntimeError( + f"{entry['name']} {entry['version']} exists on crates.io but is yanked" + ) + + expected_checksum = _sha256(_crate_file(order_file, entry)) + actual_checksum = record.get("cksum") + if actual_checksum != expected_checksum: + raise RuntimeError( + f"{entry['name']} {entry['version']} already exists on crates.io " + f"with checksum {actual_checksum}, but this build packaged " + f"{expected_checksum}" + ) + return True + + +def _validate_release_graph(metadata: dict) -> dict[str, list[str]]: + """Validate the full local dependency closure and return its edges.""" + packages = {package["name"]: package for package in metadata["packages"]} + positions = {crate: index for index, crate in enumerate(CRATES)} + dependencies: dict[str, list[str]] = {} + errors: list[str] = [] + + if len(positions) != len(CRATES): + errors.append("CRATES contains duplicate package names") + + for crate in CRATES: + package = packages.get(crate) + if package is None: + errors.append(f"{crate}: not found in workspace metadata") + continue + + allowed_registries = package.get("publish") + if allowed_registries == []: + errors.append(f"{crate}: package has publish = false") + elif ( + allowed_registries is not None + and "crates-io" not in allowed_registries + ): + errors.append( + f"{crate}: package publish list does not include crates-io" + ) + + local_dependencies: list[str] = [] + for dependency in package["dependencies"]: + dependency_name = dependency["name"] + if not dependency.get("path") or dependency_name not in packages: + continue + if dependency.get("req") in (None, "*"): + errors.append( + f"{crate} -> {dependency_name}: local dependency is missing " + "a registry version" + ) + if dependency_name not in positions: + errors.append( + f"{crate} -> {dependency_name}: local dependency is missing " + "from CRATES" + ) + continue + if positions[dependency_name] >= positions[crate]: + errors.append( + f"{crate} -> {dependency_name}: dependency must appear " + "earlier in CRATES" + ) + local_dependencies.append(dependency_name) + + dependencies[crate] = sorted( + set(local_dependencies), + key=positions.__getitem__, + ) + + if errors: + raise RuntimeError( + "Invalid crates.io release graph:\n - " + "\n - ".join(errors) + ) + return dependencies + + +def cmd_package(args: argparse.Namespace) -> int: + metadata = _cargo_metadata(args.manifest_path) + versions = { + package["name"]: package["version"] + for package in metadata["packages"] + } + try: + dependencies = _validate_release_graph(metadata) + except RuntimeError as error: + print(f"FAIL {error}") + return 1 + + target_dir = metadata["target_directory"] + package_dir = os.path.join(target_dir, "package") + out_dir = os.path.abspath(args.out_dir) + os.makedirs(out_dir, exist_ok=True) + + print(f"=== cargo package: {len(CRATES)} crates (leaf-first) ===") + for crate in CRATES: + print(f" {crate} {versions[crate]}") + print(flush=True) + + manifest = os.path.abspath(args.manifest_path) + package_args = [ + "cargo", + "package", + "--no-verify", + "--allow-dirty", + "--registry", + "crates-io", + "--manifest-path", + manifest, + ] + for crate in CRATES: + package_args += ["-p", crate] + rc = _run(package_args) + if rc != 0: + print(f"FAIL cargo package exited {rc}") + return 1 + + ordered: list[dict] = [] + for crate in CRATES: + version = versions[crate] + crate_file = f"{crate}-{version}.crate" + source = os.path.join(package_dir, crate_file) + if not os.path.isfile(source): + print(f"FAIL expected {source} was not produced by cargo package") + return 1 + shutil.copy2(source, os.path.join(out_dir, crate_file)) + ordered.append( + { + "name": crate, + "version": version, + "file": crate_file, + "dependencies": dependencies[crate], + } + ) + print(f"OK packaged {crate_file}", flush=True) + + order_path = os.path.join(out_dir, "release-order.json") + with open(order_path, "w", encoding="utf-8") as fh: + json.dump({"crates": ordered}, fh, indent=2) + print(f"\nWrote {order_path}") + print(f"=== packaged {len(ordered)} crates into {out_dir} ===") + return 0 + + +def cmd_verify_order(args: argparse.Namespace) -> int: + entries = _load_order(args.order_file) + packaged = [entry["name"] for entry in entries] + expected = json.loads(args.expected) + if packaged != expected: + print( + "Crate order mismatch between the pipeline `crateOrder` parameter " + "and the packaged release-order.json." + ) + print(f" pipeline crateOrder : {expected}") + print(f" release-order.json : {packaged}") + return 1 + + positions = {crate: index for index, crate in enumerate(packaged)} + for entry in entries: + if "dependencies" not in entry: + print(f"Crate {entry['name']} is missing dependency metadata.") + return 1 + for dependency in entry["dependencies"]: + if dependency not in positions: + print( + f"Crate {entry['name']} depends on unpackaged crate " + f"{dependency}." + ) + return 1 + if positions[dependency] >= positions[entry["name"]]: + print( + f"Crate {entry['name']} appears before dependency " + f"{dependency}." + ) + return 1 + + print(f"Crate order verified ({len(packaged)} crates, leaf-first).") + return 0 + + +def cmd_probe(_args: argparse.Namespace) -> int: + url = f"{CRATES_IO_SPARSE_INDEX}/config.json" + try: + with urllib.request.urlopen(_index_request(url), timeout=30) as response: + config = json.load(response) + if "dl" not in config: + print(f"FAIL crates.io index config at {url} has no download URL") + return 1 + except ( + urllib.error.HTTPError, + urllib.error.URLError, + TimeoutError, + json.JSONDecodeError, + ) as error: + print( + "FAIL cannot read the crates.io sparse index. Allow read-only " + f"HTTPS egress to index.crates.io on the release pool: {error}" + ) + return 1 + + print(f"OK crates.io sparse index is reachable at {url}") + return 0 + + +def cmd_stage(args: argparse.Namespace) -> int: + entry = _entry_for_crate(args.order_file, args.crate) + if entry is None: + print(f"Crate {args.crate!r} not found in {args.order_file}") + return 1 + + out_dir = os.path.abspath(args.out_dir) + if os.path.isdir(out_dir): + shutil.rmtree(out_dir) + os.makedirs(out_dir, exist_ok=True) + + source = _crate_file(args.order_file, entry) + if not os.path.isfile(source): + print(f"Crate file not found: {source}") + return 1 + shutil.copy2(source, os.path.join(out_dir, entry["file"])) + print( + f"Staged {entry['file']} ({args.crate} {entry['version']}) " + f"into {out_dir} for ESRP." + ) + return 0 + + +def cmd_verify_dependencies(args: argparse.Namespace) -> int: + entries = _load_order(args.order_file) + entry = next( + (item for item in entries if item["name"] == args.crate), + None, + ) + if entry is None: + print(f"Crate {args.crate!r} not found in {args.order_file}") + return 1 + by_name = {item["name"]: item for item in entries} + + dependencies = entry.get("dependencies") + if dependencies is None: + print(f"Crate {args.crate!r} has no dependency metadata") + return 1 + if not dependencies: + print(f"OK {args.crate} has no first-party crate dependencies.") + return 0 + + for dependency in dependencies: + dependency_entry = by_name.get(dependency) + if dependency_entry is None: + print( + f"FAIL {args.crate} depends on unpackaged crate {dependency}" + ) + return 1 + try: + published = _published_matches(args.order_file, dependency_entry) + except ( + RuntimeError, + urllib.error.HTTPError, + urllib.error.URLError, + TimeoutError, + json.JSONDecodeError, + OSError, + ) as error: + print(f"FAIL cannot validate dependency {dependency}: {error}") + return 1 + if not published: + print( + f"FAIL {args.crate} requires {dependency} " + f"{dependency_entry['version']}, but that exact package is " + "not on crates.io" + ) + return 1 + print( + f"OK {dependency} {dependency_entry['version']} is on crates.io " + "with the packaged checksum." + ) + + print( + f"=== verified {len(dependencies)} first-party dependencies for " + f"{args.crate} ===" + ) + return 0 + + +def cmd_status(args: argparse.Namespace) -> int: + entry = _entry_for_crate(args.order_file, args.crate) + if entry is None: + print(f"Crate {args.crate!r} not found in {args.order_file}") + return 1 + try: + published = _published_matches(args.order_file, entry) + except ( + RuntimeError, + urllib.error.HTTPError, + urllib.error.URLError, + TimeoutError, + json.JSONDecodeError, + OSError, + ) as error: + print(f"FAIL cannot check {args.crate} on crates.io: {error}") + return 1 + + value = "true" if published else "false" + print(f"##vso[task.setvariable variable=crateAlreadyPublished]{value}") + if published: + print( + f"OK {entry['name']} {entry['version']} is already on crates.io " + "with the packaged checksum." + ) + else: + print( + f"INFO {entry['name']} {entry['version']} is not yet on crates.io." + ) + return 0 + + +def cmd_wait(args: argparse.Namespace) -> int: + entry = _entry_for_crate(args.order_file, args.crate) + if entry is None: + print(f"Crate {args.crate!r} not found in {args.order_file}") + return 1 + crate, version = entry["name"], entry["version"] + + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + try: + if _published_matches(args.order_file, entry): + print( + f"OK {crate} {version} is live on crates.io with the " + "packaged checksum." + ) + return 0 + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as error: + print( + f"WARN crates.io index unreachable for {crate}: " + f"{error}; retrying" + ) + except (RuntimeError, json.JSONDecodeError, OSError) as error: + print(f"FAIL cannot validate {crate}: {error}") + return 1 + time.sleep(args.poll) + + print( + f"##vso[task.logissue type=error]{crate} {version} was not confirmed " + f"on crates.io within {args.timeout}s. Stopping before publishing a " + "dependent crate." + ) + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + package = sub.add_parser( + "package", + help="validate and cargo package the complete closure", + ) + package.add_argument("--manifest-path", default="src/Cargo.toml") + package.add_argument("--out-dir", required=True) + package.set_defaults(func=cmd_package) + + verify_order = sub.add_parser( + "verify-order", + help="assert crateOrder matches the packaged dependency graph", + ) + verify_order.add_argument("--order-file", required=True) + verify_order.add_argument("--expected", required=True) + verify_order.set_defaults(func=cmd_verify_order) + + probe = sub.add_parser( + "probe", + help="confirm the release pool can read the crates.io sparse index", + ) + probe.set_defaults(func=cmd_probe) + + stage = sub.add_parser( + "stage", + help="copy one crate into a clean ESRP input directory", + ) + stage.add_argument("--order-file", required=True) + stage.add_argument("--crate", required=True) + stage.add_argument("--out-dir", required=True) + stage.set_defaults(func=cmd_stage) + + dependencies = sub.add_parser( + "verify-dependencies", + help="verify one crate's first-party dependencies on crates.io", + ) + dependencies.add_argument("--order-file", required=True) + dependencies.add_argument("--crate", required=True) + dependencies.set_defaults(func=cmd_verify_dependencies) + + status = sub.add_parser( + "status", + help="set whether the exact packaged crate is already on crates.io", + ) + status.add_argument("--order-file", required=True) + status.add_argument("--crate", required=True) + status.set_defaults(func=cmd_status) + + wait = sub.add_parser( + "wait", + help="poll crates.io for the exact packaged crate", + ) + wait.add_argument("--order-file", required=True) + wait.add_argument("--crate", required=True) + wait.add_argument("--timeout", type=int, default=PROPAGATION_TIMEOUT) + wait.add_argument("--poll", type=int, default=PROPAGATION_POLL) + wait.set_defaults(func=cmd_wait) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.azure-pipelines/templates/1ES.Build.Stages.yml b/.azure-pipelines/templates/1ES.Build.Stages.yml index efa80a5af..ed8f2724e 100644 --- a/.azure-pipelines/templates/1ES.Build.Stages.yml +++ b/.azure-pipelines/templates/1ES.Build.Stages.yml @@ -14,6 +14,13 @@ parameters: type: boolean default: false + # Opt-in while multi-package `cargo package` is blocked by + # https://github.com/rust-lang/cargo/issues/17196 when crates.io is replaced + # by the private ingestion feed. + - name: packageCrates + type: boolean + default: false + - name: ESRPInfo type: object default: @@ -82,6 +89,19 @@ stages: - template: Mxc.Binary.Packaging.Job.yml +# Crate packaging is official-build-only and explicitly opt-in until Cargo +# #17196 is fixed or the packaging implementation no longer depends on it. +- ${{ if and(parameters.isOfficialBuild, parameters.packageCrates) }}: + - stage: Package_Crates + displayName: 'Package Crates' + # Source-only (no binaries), so it does not depend on Build_Binaries. Produces + # the `mxc-crates-package` artifact consumed by 1ES.Release.yml for ESRP-based + # crates.io publishing. Isolated in its own stage so it never gates npm + # packaging or the integration tests that depend on Package_MXC. + dependsOn: [] + jobs: + - template: Package.Crates.Job.yml + - stage: Lint displayName: 'Lint' dependsOn: [] diff --git a/.azure-pipelines/templates/Package.Crates.Job.yml b/.azure-pipelines/templates/Package.Crates.Job.yml new file mode 100644 index 000000000..4cdb37690 --- /dev/null +++ b/.azure-pipelines/templates/Package.Crates.Job.yml @@ -0,0 +1,58 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Packages the `mxc-sdk` crate closure into `.crate` files for ESRP-based +# crates.io publishing (consumed by 1ES.Release.yml's Publish_to_CratesIo +# stage). Publishes them as the `mxc-crates-package` pipeline artifact. +# +# Source-only: unlike the npm SDK this needs no built binaries, only a cargo +# toolchain plus the internal Mxc-Azure-Feed for dependency resolution. +# `cargo package --no-verify` tars each crate's source without compiling, while +# the helper validates that the complete first-party dependency graph is +# publishable and leaf-first. This job is currently opt-in because Cargo +# https://github.com/rust-lang/cargo/issues/17196 prevents multi-package +# packaging when crates.io is replaced by the private feed. +# +# The helper script is copied into the artifact so the release job can run its +# `stage`/`wait`/`verify-order` subcommands without checking out the source. + +parameters: + - name: cratesArtifactName + type: string + default: mxc-crates-package + +jobs: + - job: package_crates + displayName: Package Crates (crates.io) + pool: + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest + os: linux + variables: + outDir: $(Build.SourcesDirectory)/out/crates + steps: + - checkout: self + + - template: Rust.Toolchain.Public.yml@self + parameters: + targetTriple: x86_64-unknown-linux-gnu + + # Resolve dependencies through the internal feed (official build has the + # System.AccessToken needed to authenticate). Cargo #17196 currently + # causes unpublished workspace siblings to be looked up in this feed. + - template: Cargo.Setup.Private.yml@self + + - script: | + set -euo pipefail + python3 .azure-pipelines/scripts/crates_release.py package \ + --manifest-path src/Cargo.toml \ + --out-dir "$(outDir)" + cp .azure-pipelines/scripts/crates_release.py "$(outDir)/" + displayName: Package crate closure (leaf-first) + workingDirectory: $(Build.SourcesDirectory) + + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish crate package artifact + inputs: + path: $(outDir) + artifactName: ${{ parameters.cratesArtifactName }} diff --git a/.azure-pipelines/templates/Publish.CratesIo.Job.yml b/.azure-pipelines/templates/Publish.CratesIo.Job.yml new file mode 100644 index 000000000..30f2db717 --- /dev/null +++ b/.azure-pipelines/templates/Publish.CratesIo.Job.yml @@ -0,0 +1,126 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +# Publishes the `mxc-sdk` crate closure to crates.io through ESRP Release, under +# the official `microsoft-oss-releases` account. Consumes the +# `mxc-crates-package` artifact produced by the official build +# (Package.Crates.Job.yml) -- ESRP publishes pre-built `.crate` files, so no +# CARGO_REGISTRY_TOKEN ever lives in this repo. +# +# ESRP publishes via the crates.io API, which does NOT enforce dependency order +# and offers no bulk sorting (see the ESRP crates.io docs). We therefore emit +# one EsrpRelease task per crate, leaf-first. Before each publish, the job checks +# every first-party dependency directly on the crates.io sparse index; after the +# publish it confirms the exact `.crate` checksum is live before continuing. A +# run-time guard asserts the compile-time `crateOrder` matches what was packaged. + +parameters: + - name: ESRPInfo + type: object + # Leaf-first crate order. MUST match the CRATES list in + # scripts/crates_release.py; `verify-order` enforces this at run time. + - name: crateOrder + type: object + - name: cratesArtifactName + type: string + default: mxc-crates-package + # When true, run index access + stage + verify-order but SKIP ESRP publication + # and live dependency checks -- exercises the pipeline plumbing without + # publishing anything. + - name: cratesDryRun + type: boolean + default: false + +jobs: + - deployment: publish_crates + displayName: Publish crates to crates.io (ESRP) + pool: + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest + os: linux + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: MXC + artifactName: ${{ parameters.cratesArtifactName }} + targetPath: $(Pipeline.Workspace)/crates + variables: + cratesDir: $(Pipeline.Workspace)/crates + esrpDir: $(Pipeline.Workspace)/esrp-one + orderFile: $(Pipeline.Workspace)/crates/release-order.json + script: $(Pipeline.Workspace)/crates/crates_release.py + strategy: + runOnce: + deploy: + steps: + # Guard: the compile-time crate loop below and the packaged closure + # must agree, or we could publish in the wrong order / skip a crate. + - script: | + set -euo pipefail + python3 "$(script)" verify-order --order-file "$(orderFile)" --expected "$EXPECTED_ORDER" + displayName: Verify crate order matches package + env: + EXPECTED_ORDER: ${{ convertToJson(parameters.crateOrder) }} + + # Fail before publishing anything if the release pool's network + # isolation policy does not allow the crates.io sparse index. + - script: | + set -euo pipefail + python3 "$(script)" probe + displayName: Verify crates.io index access + + - ${{ each crate in parameters.crateOrder }}: + - script: | + set -euo pipefail + python3 "$(script)" stage --order-file "$(orderFile)" --crate "${{ crate }}" --out-dir "$(esrpDir)" + displayName: "Stage ${{ crate }} for ESRP" + + - ${{ if parameters.cratesDryRun }}: + - script: | + echo "##vso[task.logissue type=warning]DRY RUN: skipping ESRP publish and feed wait for ${{ crate }} (staged .crate only)." + displayName: "DRY RUN: skip publish ${{ crate }}" + - ${{ else }}: + - script: | + set -euo pipefail + python3 "$(script)" status --order-file "$(orderFile)" --crate "${{ crate }}" + displayName: "Check whether ${{ crate }} is already on crates.io" + + - script: | + set -euo pipefail + python3 "$(script)" verify-dependencies --order-file "$(orderFile)" --crate "${{ crate }}" + displayName: "Verify ${{ crate }} dependencies on crates.io" + condition: and(succeeded(), ne(variables['crateAlreadyPublished'], 'true')) + + - task: EsrpRelease@12 + displayName: "Publish ${{ crate }} to crates.io" + condition: and(succeeded(), ne(variables['crateAlreadyPublished'], 'true')) + inputs: + connectedservicename: ${{ parameters.ESRPInfo.serviceName }} + usemanagedidentity: false + keyvaultname: ${{ parameters.ESRPInfo.azureKeyVaultName }} + authcertname: ${{ parameters.ESRPInfo.authCertName }} + signcertname: ${{ parameters.ESRPInfo.signCertName }} + clientid: ${{ parameters.ESRPInfo.clientId }} + intent: 'PackageDistribution' + contenttype: 'Rust' + contentsource: 'Folder' + folderlocation: $(esrpDir) + owners: ${{ parameters.ESRPInfo.OwnersEmail }} + approvers: ${{ parameters.ESRPInfo.ApproversEmail }} + waitforreleasecompletion: true + serviceendpointurl: 'https://api.esrp.microsoft.com' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: ${{ parameters.ESRPInfo.tenantId }} + + - script: | + echo "${{ crate }} is already on crates.io with the packaged checksum; skipping ESRP." + displayName: "Skip existing ${{ crate }} version" + condition: and(succeeded(), eq(variables['crateAlreadyPublished'], 'true')) + + - script: | + set -euo pipefail + python3 "$(script)" wait --order-file "$(orderFile)" --crate "${{ crate }}" + displayName: "Wait for ${{ crate }} on crates.io" + condition: and(succeeded(), ne(variables['crateAlreadyPublished'], 'true')) diff --git a/src/Cargo.toml b/src/Cargo.toml index 8040c2027..bd0a14aed 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -55,6 +55,7 @@ strip = "debuginfo" version = "0.7.0" edition = "2021" license = "MIT" +repository = "https://github.com/microsoft/mxc" [workspace.dependencies] # Pure-Rust ETW TraceLogging provider used by `mxc_telemetry` (Windows-only; @@ -90,7 +91,7 @@ windows = { version = "0.62", features = [ "Win32_System_Time", "Win32_System_SystemServices", "Win32_System_SystemInformation", - "Win32_System_JobObjects", + "Win32_System_JobObjects", ] } windows-core = "0.62" serde = { version = "1", features = ["derive"] } @@ -101,29 +102,29 @@ base64 = "0.22" clap = { version = "4", features = ["derive"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } quick-xml = "0.41" -wxc_common = { path = "core/wxc_common" } -mxc_engine = { path = "core/mxc_engine" } +wxc_common = { path = "core/wxc_common", version = "0.7.0" } +mxc_engine = { path = "core/mxc_engine", version = "0.7.0" } mxc-sdk = { path = "core/mxc-sdk" } -appcontainer_common = { path = "backends/appcontainer/common" } -windows_sandbox_common = { path = "backends/windows_sandbox/common" } -windows_sandbox_lifecycle = { path = "backends/windows_sandbox/lifecycle" } -isolation_session_common = { path = "backends/isolation_session/common" } -hyperlight_common = { path = "backends/hyperlight/common" } -nanvix_runner = { path = "backends/nanvix/runner" } +appcontainer_common = { path = "backends/appcontainer/common", version = "0.7.0" } +windows_sandbox_common = { path = "backends/windows_sandbox/common", version = "0.7.0" } +windows_sandbox_lifecycle = { path = "backends/windows_sandbox/lifecycle", version = "0.7.0" } +isolation_session_common = { path = "backends/isolation_session/common", version = "0.7.0" } +hyperlight_common = { path = "backends/hyperlight/common", version = "0.7.0" } +nanvix_runner = { path = "backends/nanvix/runner", version = "0.7.0" } tokio = { version = "1", features = ["full"] } uuid = { version = "1", features = ["v4"] } getrandom = "0.2" libc = "0.2" nix = { version = "0.29", features = ["fs", "mount", "sched", "signal", "net", "process", "user", "term"] } -lxc_common = { path = "backends/lxc/common" } -bwrap_common = { path = "backends/bubblewrap/common" } -seatbelt_common = { path = "backends/seatbelt/common" } -wslc_common = { path = "backends/wslc/common" } -isolation_session_bindings = { path = "backends/isolation_session/bindings" } -mxc_pty = { path = "core/mxc_pty" } +lxc_common = { path = "backends/lxc/common", version = "0.7.0" } +bwrap_common = { path = "backends/bubblewrap/common", version = "0.7.0" } +seatbelt_common = { path = "backends/seatbelt/common", version = "0.7.0" } +wslc_common = { path = "backends/wslc/common", version = "0.7.0" } +isolation_session_bindings = { path = "backends/isolation_session/bindings", version = "0.7.0" } +mxc_pty = { path = "core/mxc_pty", version = "0.7.0" } flatbuffers = "25" -sandbox_spec = { path = "core/generated/base_container_specification" } -mxc_telemetry = { path = "mxc_telemetry" } +sandbox_spec = { path = "core/generated/base_container_specification", version = "0.7.0" } +mxc_telemetry = { path = "mxc_telemetry", version = "0.7.0" } widestring = "1" url = "2" winreg = "0.55" diff --git a/src/backends/appcontainer/common/Cargo.toml b/src/backends/appcontainer/common/Cargo.toml index f79e9fec4..ab209c828 100644 --- a/src/backends/appcontainer/common/Cargo.toml +++ b/src/backends/appcontainer/common/Cargo.toml @@ -3,6 +3,8 @@ name = "appcontainer_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Windows AppContainer/BaseContainer backend support for MXC sandboxing." [features] # Compile-time gate for Tier 2 (AppContainer + BFS via bfscfg.exe). diff --git a/src/backends/bubblewrap/common/Cargo.toml b/src/backends/bubblewrap/common/Cargo.toml index e337d810e..5191092d4 100644 --- a/src/backends/bubblewrap/common/Cargo.toml +++ b/src/backends/bubblewrap/common/Cargo.toml @@ -3,6 +3,8 @@ name = "bwrap_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Bubblewrap (Linux) backend support for MXC sandboxing." [dependencies] wxc_common = { workspace = true } diff --git a/src/backends/hyperlight/common/Cargo.toml b/src/backends/hyperlight/common/Cargo.toml index 6e5ceebaa..5a6fda700 100644 --- a/src/backends/hyperlight/common/Cargo.toml +++ b/src/backends/hyperlight/common/Cargo.toml @@ -3,6 +3,8 @@ name = "hyperlight_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Hyperlight micro-VM backend support for the MXC execution engine." [dependencies] wxc_common = { workspace = true } diff --git a/src/backends/isolation_session/bindings/Cargo.toml b/src/backends/isolation_session/bindings/Cargo.toml index ed4ccee0c..bfd07fbe8 100644 --- a/src/backends/isolation_session/bindings/Cargo.toml +++ b/src/backends/isolation_session/bindings/Cargo.toml @@ -3,7 +3,8 @@ name = "isolation_session_bindings" version.workspace = true edition.workspace = true license.workspace = true -publish = false +repository.workspace = true +description = "Windows IsolationSession API bindings used by MXC." [dependencies] windows-core = { workspace = true } diff --git a/src/backends/isolation_session/common/Cargo.toml b/src/backends/isolation_session/common/Cargo.toml index 97bd7a2b8..9ca93c9e5 100644 --- a/src/backends/isolation_session/common/Cargo.toml +++ b/src/backends/isolation_session/common/Cargo.toml @@ -3,6 +3,8 @@ name = "isolation_session_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Windows IsolationSession backend support for the MXC execution engine." [dependencies] wxc_common = { workspace = true } diff --git a/src/backends/lxc/common/Cargo.toml b/src/backends/lxc/common/Cargo.toml index 5bf54ca70..18800b5a9 100644 --- a/src/backends/lxc/common/Cargo.toml +++ b/src/backends/lxc/common/Cargo.toml @@ -3,6 +3,8 @@ name = "lxc_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Shared Linux container backend support for MXC sandboxing." [dependencies] wxc_common = { workspace = true } diff --git a/src/backends/nanvix/common/Cargo.toml b/src/backends/nanvix/common/Cargo.toml index 264a1c287..522b69f47 100644 --- a/src/backends/nanvix/common/Cargo.toml +++ b/src/backends/nanvix/common/Cargo.toml @@ -3,6 +3,7 @@ name = "nanvix_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true description = "Shared constants and configuration types for NanVix micro-VM binaries" [dependencies] diff --git a/src/backends/nanvix/runner/Cargo.toml b/src/backends/nanvix/runner/Cargo.toml index 081942e21..d5593b7eb 100644 --- a/src/backends/nanvix/runner/Cargo.toml +++ b/src/backends/nanvix/runner/Cargo.toml @@ -3,10 +3,12 @@ name = "nanvix_runner" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "NanVix micro-VM runner used by the MXC execution engine." [dependencies] wxc_common = { workspace = true, features = ["microvm"] } -nanvix_common = { path = "../common" } +nanvix_common = { path = "../common", version = "0.7.0" } uuid = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] diff --git a/src/backends/seatbelt/common/Cargo.toml b/src/backends/seatbelt/common/Cargo.toml index fb7029ae6..9b5248f03 100644 --- a/src/backends/seatbelt/common/Cargo.toml +++ b/src/backends/seatbelt/common/Cargo.toml @@ -3,6 +3,8 @@ name = "seatbelt_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Seatbelt (macOS) backend support for MXC sandboxing." [dependencies] # Provides ExecutionRequest, ContainerPolicy, SeatbeltConfig, Logger, and diff --git a/src/backends/windows_sandbox/common/Cargo.toml b/src/backends/windows_sandbox/common/Cargo.toml index 617a5dd23..3fe1e3f54 100644 --- a/src/backends/windows_sandbox/common/Cargo.toml +++ b/src/backends/windows_sandbox/common/Cargo.toml @@ -3,6 +3,8 @@ name = "windows_sandbox_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Shared protocol and state types for the MXC Windows Sandbox backend." [dependencies] wxc_common = { workspace = true } diff --git a/src/backends/windows_sandbox/lifecycle/Cargo.toml b/src/backends/windows_sandbox/lifecycle/Cargo.toml index 48cfcc67f..9e96c0c64 100644 --- a/src/backends/windows_sandbox/lifecycle/Cargo.toml +++ b/src/backends/windows_sandbox/lifecycle/Cargo.toml @@ -2,6 +2,9 @@ name = "windows_sandbox_lifecycle" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Host-side lifecycle management for the MXC Windows Sandbox backend." [dependencies] windows_sandbox_common = { workspace = true } diff --git a/src/backends/wslc/common/Cargo.toml b/src/backends/wslc/common/Cargo.toml index 23423f0fa..c627324c8 100644 --- a/src/backends/wslc/common/Cargo.toml +++ b/src/backends/wslc/common/Cargo.toml @@ -3,6 +3,8 @@ name = "wslc_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Windows Subsystem for Linux container backend support for MXC." [features] default = [] diff --git a/src/core/generated/base_container_specification/Cargo.toml b/src/core/generated/base_container_specification/Cargo.toml index 6765005ee..c99c6f54f 100644 --- a/src/core/generated/base_container_specification/Cargo.toml +++ b/src/core/generated/base_container_specification/Cargo.toml @@ -3,7 +3,7 @@ name = "sandbox_spec" version.workspace = true edition.workspace = true license.workspace = true -publish = false +repository.workspace = true description = "Generated FlatBuffers bindings for SandboxSpec" [dependencies] diff --git a/src/core/mxc-sdk/Cargo.toml b/src/core/mxc-sdk/Cargo.toml index df13444b5..e1626e6eb 100644 --- a/src/core/mxc-sdk/Cargo.toml +++ b/src/core/mxc-sdk/Cargo.toml @@ -3,6 +3,7 @@ name = "mxc-sdk" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true description = "Importable library for starting MXC sandboxes in-process (no pty, streaming stdio)." [lib] diff --git a/src/core/mxc_engine/Cargo.toml b/src/core/mxc_engine/Cargo.toml index ca32a0c8f..11aefae85 100644 --- a/src/core/mxc_engine/Cargo.toml +++ b/src/core/mxc_engine/Cargo.toml @@ -3,6 +3,7 @@ name = "mxc_engine" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true description = "MXC execution engine: backend dispatch, host probing, and config building shared by the public SDK and the executor binaries." [lib] diff --git a/src/core/mxc_pty/Cargo.toml b/src/core/mxc_pty/Cargo.toml index cba91f420..82b08b701 100644 --- a/src/core/mxc_pty/Cargo.toml +++ b/src/core/mxc_pty/Cargo.toml @@ -3,6 +3,8 @@ name = "mxc_pty" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Unix pty helpers for MXC sandbox executors." # Unix-only at runtime: mxc_pty wraps `openpty` + `pre_exec` (TIOCSCTTY, # setsid) + sigwait-style helpers, none of which have a sensible Windows diff --git a/src/core/wxc_common/Cargo.toml b/src/core/wxc_common/Cargo.toml index 09f2e3ee7..ded7b2402 100644 --- a/src/core/wxc_common/Cargo.toml +++ b/src/core/wxc_common/Cargo.toml @@ -3,6 +3,8 @@ name = "wxc_common" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true +description = "Shared core types (execution requests, policies, wire model) for MXC sandboxing." [features] microvm = ["dep:nanvix_common", "dep:uuid"] @@ -19,7 +21,7 @@ url = { workspace = true } getrandom = { workspace = true } semver = "1" schemars = { version = "0.8", optional = true } -nanvix_common = { path = "../../backends/nanvix/common", optional = true } +nanvix_common = { path = "../../backends/nanvix/common", version = "0.7.0", optional = true } uuid = { workspace = true, optional = true } mxc_telemetry = { workspace = true } diff --git a/src/mxc_telemetry/Cargo.toml b/src/mxc_telemetry/Cargo.toml index fa5d8a47f..d1a7ef657 100644 --- a/src/mxc_telemetry/Cargo.toml +++ b/src/mxc_telemetry/Cargo.toml @@ -3,6 +3,7 @@ name = "mxc_telemetry" version.workspace = true edition.workspace = true license.workspace = true +repository.workspace = true description = "Pure Rust TraceLogging ETW telemetry for MXC" [target.'cfg(windows)'.dependencies]