Skip to content

Use partial fetch (blob:none) for checkpoints metadata #712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pjbgf merged 3 commits into from
Apr 9, 2026
Merged

Use partial fetch (blob:none) for checkpoints metadata #712

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fac07a0
Use partial fetch (blob:none) for metadata session sync
pjbgf Mar 17, 2026
4c4dd46
Use partial clone for all checkpoint fetches
pjbgf Apr 9, 2026
73bea7b
Remove partial clone optimisation from user message
pjbgf Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/entire/cli/checkpoint/v2_read.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ func (s *V2GitStore) fetchRemoteFullRefs(ctx context.Context) error {
return nil
}

args := append([]string{"fetch", s.FetchRemote}, refSpecs...)
args := append([]string{"fetch", "--no-tags", s.FetchRemote}, refSpecs...)
fetchCmd := exec.CommandContext(ctx, "git", args...)
if fetchOutput, fetchErr := fetchCmd.CombinedOutput(); fetchErr != nil {
return fmt.Errorf("fetch failed: %s", fetchOutput)
Expand Down
18 changes: 11 additions & 7 deletions cmd/entire/cli/git_operations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,8 @@ func FetchAndCheckoutRemoteBranch(ctx context.Context, branchName string) error

// FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin and creates/updates the local branch.
// This is used when the metadata branch exists on remote but not locally.
// The fetch is treeless (--filter=blob:none) because checkpoint metadata reads
// support on-demand blob retrieval.
// Uses git CLI instead of go-git for fetch because go-git doesn't use credential helpers,
// which breaks HTTPS URLs that require authentication.
func FetchMetadataBranch(ctx context.Context) error {
Expand All @@ -383,7 +385,7 @@ func FetchMetadataBranch(ctx context.Context) error {

refSpec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", branchName, branchName)

fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "origin", refSpec)
fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--no-tags", "--filter=blob:none", "origin", refSpec)
if output, err := fetchCmd.CombinedOutput(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
return errors.New("fetch timed out after 2 minutes")
Expand Down Expand Up @@ -425,7 +427,7 @@ func FetchMetadataTreeOnly(ctx context.Context) error {

refSpec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", branchName, branchName)

fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--depth=1", "--filter=blob:none", "origin", refSpec)
fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--no-tags", "--depth=1", "--filter=blob:none", "origin", refSpec)
if output, err := fetchCmd.CombinedOutput(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
return errors.New("treeless fetch timed out after 2 minutes")
Expand Down Expand Up @@ -463,7 +465,7 @@ func FetchV2MainTreeOnly(ctx context.Context) error {

refSpec := fmt.Sprintf("+%s:%s", paths.V2MainRefName, paths.V2MainRefName)

fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--depth=1", "--filter=blob:none", "origin", refSpec)
fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--no-tags", "--depth=1", "--filter=blob:none", "origin", refSpec)
if output, err := fetchCmd.CombinedOutput(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
return errors.New("v2 treeless fetch timed out after 2 minutes")
Expand All @@ -474,15 +476,17 @@ func FetchV2MainTreeOnly(ctx context.Context) error {
return nil
}

// FetchV2MainRef fetches the v2 /main ref from origin (full fetch, including blobs).
// FetchV2MainRef fetches the v2 /main ref from origin.
// The fetch is treeless (--filter=blob:none) because /main is metadata-only and
// v2 checkpoint reads handle transcript retrieval separately.
// Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/.
func FetchV2MainRef(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()

refSpec := fmt.Sprintf("+%s:%s", paths.V2MainRefName, paths.V2MainRefName)

fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "origin", refSpec)
fetchCmd := strategy.CheckpointGitCommand(ctx, "origin", "fetch", "--no-tags", "--filter=blob:none", "origin", refSpec)
if output, err := fetchCmd.CombinedOutput(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
return errors.New("v2 fetch timed out after 2 minutes")
Expand Down Expand Up @@ -543,9 +547,9 @@ func FetchBlobsByHash(ctx context.Context, hashes []plumbing.Hash) error {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()

// Build fetch args: "git fetch origin <hash1> <hash2> ..."
// Build fetch args: "git fetch --no-tags origin <hash1> <hash2> ..."
// This uses the normal transport + credential helpers, unlike fetch-pack.
args := []string{"fetch", "--no-write-fetch-head", "origin"}
args := []string{"fetch", "--no-tags", "--no-write-fetch-head", "origin"}
for _, h := range hashes {
args = append(args, h.String())
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/entire/cli/integration_test/testenv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1901,7 +1901,7 @@ func (env *TestEnv) FetchMetadataBranch(remoteURL string) {

branchName := paths.MetadataBranchName
refSpec := "+refs/heads/" + branchName + ":refs/heads/" + branchName
cmd := exec.CommandContext(env.T.Context(), "git", "fetch", "--no-tags", remoteURL, refSpec)
cmd := exec.CommandContext(env.T.Context(), "git", "fetch", "--no-tags", "--filter=blob:none", remoteURL, refSpec)
cmd.Dir = env.RepoDir
cmd.Env = testutil.GitIsolatedEnv()

Expand Down
4 changes: 2 additions & 2 deletions cmd/entire/cli/strategy/checkpoint_remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ func FetchMetadataBranch(ctx context.Context, remoteURL string) error {

tmpRef := "refs/entire-fetch-tmp/" + branchName
refSpec := fmt.Sprintf("+refs/heads/%s:%s", branchName, tmpRef)
fetchCmd := CheckpointGitCommand(fetchCtx, remoteURL, "fetch", "--no-tags", remoteURL, refSpec)
fetchCmd := CheckpointGitCommand(fetchCtx, remoteURL, "fetch", "--no-tags", "--filter=blob:none", remoteURL, refSpec)
// Merge GIT_TERMINAL_PROMPT=0 into whatever env CheckpointGitCommand set.
// If the token was injected, cmd.Env is already populated; otherwise use os.Environ().
if fetchCmd.Env == nil {
Expand Down Expand Up @@ -414,7 +414,7 @@ func FetchV2MainFromURL(ctx context.Context, remoteURL string) error {
defer cancel()

refSpec := fmt.Sprintf("+%s:%s", paths.V2MainRefName, paths.V2MainRefName)
fetchCmd := CheckpointGitCommand(fetchCtx, remoteURL, "fetch", "--no-tags", remoteURL, refSpec)
fetchCmd := CheckpointGitCommand(fetchCtx, remoteURL, "fetch", "--no-tags", "--filter=blob:none", remoteURL, refSpec)
if fetchCmd.Env == nil {
fetchCmd.Env = os.Environ()
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/entire/cli/strategy/checkpoint_token_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ func TestCheckpointToken_GIT_TERMINAL_PROMPT_Coexistence(t *testing.T) {
t.Setenv(CheckpointTokenEnvVar, "coexist-token")

cmd := CheckpointGitCommand(context.Background(), "https://github.com/org/repo.git",
"fetch", "--no-tags", "https://github.com/org/repo.git", "refs/heads/main")
"fetch", "--no-tags", "--filter=blob:none", "https://github.com/org/repo.git", "refs/heads/main")
require.NotNil(t, cmd.Env)

// Simulate what fetchMetadataBranchIfMissing does: append GIT_TERMINAL_PROMPT
Expand Down
7 changes: 5 additions & 2 deletions cmd/entire/cli/strategy/push_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,11 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string
fetchedRefName = plumbing.NewRemoteReferenceName(target, branchName)
}

// Use git CLI for fetch (go-git's fetch can be tricky with auth)
fetchCmd := CheckpointGitCommand(ctx, target, "fetch", target, refSpec)
// Use git CLI for fetch (go-git's fetch can be tricky with auth).
// Use --filter=blob:none for a partial fetch that downloads only commits
// and trees, skipping blobs. The merge only needs the tree structure to
// combine entries; blobs are already local or fetched on demand.
fetchCmd := CheckpointGitCommand(ctx, target, "fetch", "--no-tags", "--filter=blob:none", target, refSpec)
if output, err := fetchCmd.CombinedOutput(); err != nil {
return fmt.Errorf("fetch failed: %s", output)
}
Expand Down
7 changes: 5 additions & 2 deletions cmd/entire/cli/strategy/push_v2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ func fetchAndMergeRef(ctx context.Context, target string, refName plumbing.Refer
tmpRefName := plumbing.ReferenceName("refs/entire-fetch-tmp/" + tmpRefSuffix)
refSpec := fmt.Sprintf("+%s:%s", refName, tmpRefName)

fetchCmd := CheckpointGitCommand(ctx, target, "fetch", target, refSpec)
// Use --filter=blob:none for a partial fetch that downloads only commits
// and trees, skipping blobs. The merge only needs the tree structure to
// combine entries; blobs are already local or fetched on demand.
fetchCmd := CheckpointGitCommand(ctx, target, "fetch", "--no-tags", "--filter=blob:none", target, refSpec)
fetchCmd.Env = append(fetchCmd.Env, "GIT_TERMINAL_PROMPT=0")
if output, err := fetchCmd.CombinedOutput(); err != nil {
return fmt.Errorf("fetch failed: %s", output)
Expand Down Expand Up @@ -244,7 +247,7 @@ func handleRotationConflict(ctx context.Context, target string, repo *git.Reposi
// Fetch the latest archived generation
archiveTmpRef := plumbing.ReferenceName("refs/entire-fetch-tmp/archive-" + latestArchive)
archiveRefSpec := fmt.Sprintf("+%s:%s", archiveRefName, archiveTmpRef)
fetchCmd := CheckpointGitCommand(ctx, target, "fetch", target, archiveRefSpec)
fetchCmd := CheckpointGitCommand(ctx, target, "fetch", "--no-tags", "--filter=blob:none", target, archiveRefSpec)
fetchCmd.Env = append(fetchCmd.Env, "GIT_TERMINAL_PROMPT=0")
if output, fetchErr := fetchCmd.CombinedOutput(); fetchErr != nil {
return fmt.Errorf("fetch archived generation failed: %s", output)
Expand Down
Loading