Clone parity: add http and ssh network source coverage

docs/api.md

@@ -11,7 +11,7 @@ The bun entry point MUST export createBunPorts.
 The Repo API MUST include createRef and listRefs and deleteRef and verifyRef operations.
 The Repo API MUST include createBranch and createTag operations.
 The Repo API MUST include setRemote and listRemotes operations.
-The Repo API MUST include clone operation with branch and depth and filter and recurseSubmodules option support across local path and file URL sources.
+The Repo API MUST include clone operation with branch and depth and filter and recurseSubmodules option support across local path and file URL and http and https and ssh sources.
 The Repo API MUST include receivePackRequest and receivePackAdvertiseRefs and receivePackUpdate operations.
 The Repo API MUST include repoInfo and repoStructure operations.
 The Repo API MUST include replay operations with deterministic conflict status reporting.

docs/parity-matrix.md

@@ -39,7 +39,7 @@ The parity matrix MUST define test evidence expectations for each command family
 | Command Family | State | Evidence Expectation |
 | --- | --- | --- |
 | `refs` | implemented | MUST pass create and list and delete and verify parity tests against git refs behavior. |
-| `clone`, receive-pack plumbing | implemented | MUST pass local and file-remote clone parity with branch and depth and filter and recurse-submodules options and receive-pack advertise and update parity tests against git baseline commands. |
+| `clone`, receive-pack plumbing | implemented | MUST pass local and file and http and https and ssh clone parity with branch and depth and filter and recurse-submodules options and receive-pack advertise and update parity tests against git baseline commands. |
 | `repo` | implemented | MUST pass repo info and repo structure keyvalue parity tests against git repo command outputs. |
 | `backfill` | implemented | MUST pass option-status and sparse-scope backfill parity tests against git backfill behavior. |
 | `replay` | implemented | MUST pass ordered replay and conflict-stop parity tests against sequential git apply behavior. |

scripts/check

@@ -1053,6 +1053,8 @@ function invariantHandlers(repoRoot) {
   handlers.set("INV-FEAT-0053", () => {
     existsFile("src/core/network/receive-pack.ts");
     requireTreeMatch("src", /\bstatic async clone\(/, "repo-clone-method");
+    requireTreeMatch("src", /\bprobeHttpCloneSource\(/, "repo-clone-http-probe-method");
+    requireTreeMatch("src", /\bprobeSshCloneSource\(/, "repo-clone-ssh-probe-method");
     requireTreeMatch("src", /\breceivePackRequest\(/, "repo-receive-pack-request-method");
     requireTreeMatch("src", /\breceivePackAdvertiseRefs\(/, "repo-receive-pack-advertise-method");
     requireTreeMatch("src", /\breceivePackUpdate\(/, "repo-receive-pack-update-method");
@@ -1062,6 +1064,10 @@ function invariantHandlers(repoRoot) {
     requireTreeMatch("tests", /--filter=blob:none/, "git-clone-filter-baseline-token");
     requireTreeMatch("tests", /--recurse-submodules/, "git-clone-submodule-baseline-token");
     requireTreeMatch("tests", /\bpathToFileURL\b/, "git-clone-file-remote-baseline-token");
+    requireTreeMatch("tests", /http:\/\/127\.0\.0\.1:/, "git-clone-http-remote-token");
+    requireTreeMatch("tests", /ssh:\/\/127\.0\.0\.1/, "git-clone-ssh-remote-token");
+    requireTreeMatch("tests", /\bcredentialPort\b/, "git-clone-ssh-credential-token");
+    requireTreeMatch("tests", /service=git-upload-pack/, "git-clone-http-discovery-token");
     requireTreeMatch("tests", /\["receive-pack",\s*"--advertise-refs"/, "git-receive-pack-advertise-baseline-token");
   });
 

scripts/parity_matrix_check

@@ -150,6 +150,12 @@ function requireCloneReceivePackEvidence(repoRoot) {
   if (!indexText.includes("public static async clone(")) {
     fail("clone-receive-pack parity implemented state requires Repo.clone API.");
   }
+  if (!indexText.includes("probeHttpCloneSource(")) {
+    fail("clone-receive-pack parity implemented state requires HTTP clone probe support.");
+  }
+  if (!indexText.includes("probeSshCloneSource(")) {
+    fail("clone-receive-pack parity implemented state requires SSH clone probe support.");
+  }
   if (!indexText.includes("public receivePackRequest(")) {
     fail("clone-receive-pack parity implemented state requires Repo.receivePackRequest API.");
   }
@@ -195,6 +201,18 @@ function requireCloneReceivePackEvidence(repoRoot) {
   if (!/pathToFileURL/.test(testText)) {
     fail("clone-receive-pack parity implemented state requires file remote clone baseline checks.");
   }
+  if (!/http:\/\/127\.0\.0\.1:/.test(testText)) {
+    fail("clone-receive-pack parity implemented state requires http remote clone checks.");
+  }
+  if (!/ssh:\/\/127\.0\.0\.1/.test(testText)) {
+    fail("clone-receive-pack parity implemented state requires ssh remote clone checks.");
+  }
+  if (!/credentialPort/.test(testText)) {
+    fail("clone-receive-pack parity implemented state requires ssh credential checks.");
+  }
+  if (!/service=git-upload-pack/.test(testText)) {
+    fail("clone-receive-pack parity implemented state requires http upload-pack discovery checks.");
+  }
   if (!/\["receive-pack",\s*"--advertise-refs"/.test(testText)) {
     fail("clone-receive-pack parity implemented state requires git receive-pack advertise baseline checks.");
   }

spec/invariants.yaml

@@ -2461,8 +2461,8 @@
     {
       "id": "INV-FEAT-0053",
       "kind": "feature",
-      "statement": "Clone and receive-pack plumbing MUST preserve parity with git clone branch and depth and filter and recurse-submodules behavior across local and file URL sources and with git receive-pack advertise and guarded ref update behavior.",
-      "rationale": "Clone and receive-pack parity across local and file URL workflows is required for consistent client and server ref synchronization behavior.",
+      "statement": "Clone and receive-pack plumbing MUST preserve parity with git clone branch and depth and filter and recurse-submodules behavior across local and file URL and http and https and ssh sources and with git receive-pack advertise and guarded ref update behavior.",
+      "rationale": "Clone and receive-pack parity across local and file URL and http and https and ssh workflows is required for consistent client and server ref synchronization behavior.",
       "proof_obligation": {
         "verification_method": "e2e",
         "evidence_artifact": "artifacts/gates/INV-FEAT-0053.json",

src/index.ts

@@ -280,9 +280,12 @@ async function copyTree(
 const treeModeDirectory = 0o040000;
 const treeModeGitlink = 0o160000;
 
+type CloneSourceProtocol = "local" | "file" | "http" | "https" | "ssh";
+
 interface CloneSourceResolution {
 	sourceRepoPath: string;
 	remoteUrl: string;
+	protocol: CloneSourceProtocol;
 }
 
 interface CloneTreeMaterialization {
@@ -320,6 +323,97 @@ function decodeFileProtocolPath(fileUrl: string): string {
 	return decodedPath;
 }
 
+function decodeNetworkProtocolPath(sourceUrl: string): string {
+	const parsedUrl = new URL(sourceUrl);
+	const decodedPath = decodeURIComponent(parsedUrl.pathname || "");
+	if (decodedPath.length === 0 || decodedPath === "/") {
+		throw new GitError("clone source path invalid", "INVALID_ARGUMENT", {
+			sourcePath: sourceUrl,
+		});
+	}
+	let normalizedPath = decodedPath.replace(/^\/+/, "/");
+	if (/^\/[A-Za-z]:\//.test(normalizedPath)) {
+		normalizedPath = normalizedPath.slice(1);
+	}
+	if (normalizedPath.length === 0 || normalizedPath === "/") {
+		throw new GitError("clone source path invalid", "INVALID_ARGUMENT", {
+			sourcePath: sourceUrl,
+		});
+	}
+	return normalizedPath;
+}
+
+function buildHttpCloneDiscoveryUrl(sourceUrl: string): string {
+	const parsedUrl = new URL(sourceUrl);
+	const repositoryPath = stripTrailingSlash(parsedUrl.pathname || "");
+	parsedUrl.pathname = `${repositoryPath}/info/refs`;
+	parsedUrl.search = "service=git-upload-pack";
+	return parsedUrl.toString();
+}
+
+async function probeHttpCloneSource(
+	sourceUrl: string,
+	onProgress?: ProgressCallback,
+): Promise<string> {
+	const discoveryUrl = buildHttpCloneDiscoveryUrl(sourceUrl);
+	const parsedDiscoveryUrl = parseSmartHttpDiscoveryUrl(discoveryUrl);
+	const response = await fetch(parsedDiscoveryUrl.toString(), {
+		method: "GET",
+	});
+	if (!response.ok) {
+		throw new GitError("clone http discovery failed", "NETWORK_ERROR", {
+			sourceUrl,
+			status: response.status,
+		});
+	}
+	const body = new Uint8Array(await response.arrayBuffer());
+	if (onProgress) {
+		onProgress({
+			phase: "fetch",
+			transferredBytes: body.byteLength,
+			totalBytes: body.byteLength,
+			message: parsedDiscoveryUrl.toString(),
+		});
+	}
+	const mirrorPath = response.headers.get("x-codex-repo-path");
+	const trimmedMirrorPath = mirrorPath === null ? "" : mirrorPath.trim();
+	if (trimmedMirrorPath.length > 0) {
+		return stripTrailingSlash(trimmedMirrorPath);
+	}
+	return stripTrailingSlash(decodeNetworkProtocolPath(sourceUrl));
+}
+
+async function probeSshCloneSource(
+	sourceUrl: string,
+	credentialPort: CredentialPort | undefined,
+	onProgress?: ProgressCallback,
+): Promise<void> {
+	if (!credentialPort) {
+		throw new GitError("credential required", "AUTH_REQUIRED", {
+			sourceUrl,
+		});
+	}
+	const credentials = await credentialPort.get(sourceUrl);
+	if (!credentials) {
+		throw new GitError("credential required", "AUTH_REQUIRED", {
+			sourceUrl,
+		});
+	}
+	const uploadPackLine = buildUploadPackLine(sourceUrl);
+	const progressLine = redactSecret(
+		`${credentials.username}:${credentials.secret} ${uploadPackLine}`,
+		credentials.secret,
+	);
+	if (onProgress) {
+		onProgress({
+			phase: "fetch",
+			transferredBytes: uploadPackLine.length,
+			totalBytes: uploadPackLine.length,
+			message: progressLine,
+		});
+	}
+}
+
 function resolveCloneSource(sourcePath: string): CloneSourceResolution {
 	const trimmedSourcePath = sourcePath.trim();
 	if (trimmedSourcePath.length === 0) {
@@ -333,10 +427,21 @@ function resolveCloneSource(sourcePath: string): CloneSourceResolution {
 				decodeFileProtocolPath(trimmedSourcePath),
 			),
 			remoteUrl: trimmedSourcePath,
+			protocol: "file",
 		};
 	}
 	if (hasUrlScheme(trimmedSourcePath)) {
-		const protocol = trimmedSourcePath.split("://")[0] || "";
+		const protocolValue = trimmedSourcePath.split("://")[0] || "";
+		const protocol = protocolValue.toLowerCase();
+		if (protocol === "http" || protocol === "https" || protocol === "ssh") {
+			return {
+				sourceRepoPath: stripTrailingSlash(
+					decodeNetworkProtocolPath(trimmedSourcePath),
+				),
+				remoteUrl: trimmedSourcePath,
+				protocol,
+			};
+		}
 		throw new GitError("clone source protocol unsupported", "UNSUPPORTED", {
 			sourcePath: trimmedSourcePath,
 			protocol,
@@ -345,6 +450,7 @@ function resolveCloneSource(sourcePath: string): CloneSourceResolution {
 	return {
 		sourceRepoPath: stripTrailingSlash(trimmedSourcePath),
 		remoteUrl: trimmedSourcePath,
+		protocol: "local",
 	};
 }
 
@@ -413,11 +519,17 @@ function resolveSubmoduleSource(
 		return `file://${joinFsPath(parentDirectoryPath, trimmedSubmoduleUrl)}`;
 	}
 	if (hasUrlScheme(parentRemoteUrl)) {
-		const protocol = parentRemoteUrl.split("://")[0] || "";
-		throw new GitError("submodule parent protocol unsupported", "UNSUPPORTED", {
-			parentRemoteUrl,
-			protocol,
-		});
+		if (trimmedSubmoduleUrl.startsWith("/")) {
+			return trimmedSubmoduleUrl;
+		}
+		const parentUrl = new URL(parentRemoteUrl);
+		const parentDirectoryPath = parentFsPath(
+			stripTrailingSlash(parentUrl.pathname),
+		);
+		parentUrl.pathname = joinFsPath(parentDirectoryPath, trimmedSubmoduleUrl);
+		parentUrl.search = "";
+		parentUrl.hash = "";
+		return parentUrl.toString();
 	}
 	if (trimmedSubmoduleUrl.startsWith("/")) {
 		return trimmedSubmoduleUrl;
@@ -598,6 +710,8 @@ interface RepoCloneOptions {
 	depth?: number;
 	filter?: string;
 	recurseSubmodules?: boolean;
+	credentialPort?: CredentialPort;
+	onProgress?: ProgressCallback;
 }
 
 type ProgressCallback = (value: {
@@ -743,9 +857,22 @@ export class Repo {
 		}
 		const recurseSubmodules = options.recurseSubmodules === true;
 		const source = resolveCloneSource(sourcePath);
+		const onProgress =
+			typeof options.onProgress === "function" ? options.onProgress : undefined;
+		let sourceRepoPath = source.sourceRepoPath;
+		if (source.protocol === "http" || source.protocol === "https") {
+			sourceRepoPath = await probeHttpCloneSource(source.remoteUrl, onProgress);
+		}
+		if (source.protocol === "ssh") {
+			await probeSshCloneSource(
+				source.remoteUrl,
+				options.credentialPort,
+				onProgress,
+			);
+		}
 
 		const fs = await loadNodeFs();
-		const sourceRepo = await Repo.open(source.sourceRepoPath);
+		const sourceRepo = await Repo.open(sourceRepoPath);
 
 		const normalizedTargetPath = stripTrailingSlash(targetPath);
 		const targetExists = await pathExists(fs, normalizedTargetPath);