Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions eng/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,14 @@ installing `git-go-patch`:
* `pwsh eng/run.ps1 submodule-refresh` updates the submodule and applies the
patches.
* Pass `-commits` to apply each patch as a separate commit.
* Pass `-ref <commit-or-ref>` to fetch and use a different upstream revision.
Add `-origin <repository-url>` to fetch it from another repository, or
`-shallow` to limit the fetch to depth 1.
This stages the selected `go` submodule commit in the outer repository so
subsequent refreshes keep using it. To return to the committed revision,
run `git restore --staged -- go` from the repository root, then refresh again.
Refreshing discards changes in the submodule; don't commit the staged override
unless you intend to update the recorded submodule commit.
* `pwsh eng/run.ps1 build -refresh` refreshes the submodule and applies patches
and then goes on to build the Microsoft build of Go.

Expand Down
74 changes: 69 additions & 5 deletions eng/_util/cmd/submodule-refresh/submodule-refresh.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ import (
"io"
"log"
"os"
"os/exec"
"path/filepath"

"github.com/microsoft/go-infra/executil"
"github.com/microsoft/go-infra/patch"
"github.com/microsoft/go-infra/submodule"
"github.com/microsoft/go/_util/buildutil"
Expand All @@ -28,6 +30,7 @@ var (
skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.")
take = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.")
origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
ref = flag.String("ref", "", "Fetch this commit or ref instead of the recorded submodule commit, and stage it in the outer repository so subsequent refreshes use it rather than restoring the original commit, desirable in some CI scenarios.")
shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
)
Expand Down Expand Up @@ -58,14 +61,18 @@ func main() {
}

func refresh(rootDir string) error {
if err := buildutil.Retry(3, func() error {
return submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow)
}); err != nil {
config, err := patch.FindAncestorConfig(rootDir)
if err != nil {
return err
}

config, err := patch.FindAncestorConfig(rootDir)
if err != nil {
if *ref != "" {
if err := initRef(config.RootDir, config.SubmoduleDir); err != nil {
Comment thread
qmuntal marked this conversation as resolved.
Resolved
return err
}
} else if err := buildutil.Retry(3, func() error {
return submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow)
}); err != nil {
return err
}

Expand Down Expand Up @@ -117,6 +124,63 @@ func refresh(rootDir string) error {
return nil
}

func initRef(rootDir, submodulePath string) error {
// Register the submodule without fetching the recorded submodule commit.
if err := executil.Run(executil.Dir(rootDir, "git", "submodule", "init", "--", submodulePath)); err != nil {
return err
}
source := *origin
if source == "" {
var err error
source, err = executil.SpaceTrimmedCombinedOutput(executil.Dir(rootDir, "git", "config", "--get", "submodule.go.url"))
if err != nil {
return err
}
}
submoduleDir := filepath.Join(rootDir, submodulePath)
if err := executil.Run(executil.Dir(rootDir, "git", "init", "--", submoduleDir)); err != nil {
return err
}
// A later refresh may need to fetch the recorded submodule commit from this remote.
if err := executil.RunQuiet(executil.Dir(submoduleDir, "git", "config", "--get", "remote.origin.url")); err != nil {
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
return err
}
if err := executil.Run(executil.Dir(submoduleDir, "git", "remote", "add", "origin", source)); err != nil {
return err
}
}
args := []string{}
if *fetchBearerToken != "" {
args = append(args, "-c", "http.extraheader=AUTHORIZATION: bearer "+*fetchBearerToken)
}
args = append(args, "fetch", "--no-tags")
if *shallow {
args = append(args, "--depth=1")
}
args = append(args, "--", source, *ref)
if err := buildutil.Retry(3, func() error {
return executil.Run(executil.Dir(submoduleDir, "git", args...))
}); err != nil {
return err
}
commit, err := executil.SpaceTrimmedCombinedOutput(executil.Dir(submoduleDir, "git", "rev-parse", "--verify", "FETCH_HEAD^{commit}"))
if err != nil {
return err
}
if err := executil.Run(executil.Dir(submoduleDir, "git", "checkout", "--detach", "--force", commit)); err != nil {
return err
}
// Reset and later refreshes run "git submodule update", which checks out the
// commit recorded in the outer index. Stage the override so they don't
// restore the original commit before applying patches.
if err := executil.Run(executil.Dir(rootDir, "git", "update-index", "--cacheinfo", "160000", commit, submodulePath)); err != nil {
return err
}
return executil.Run(executil.Dir(rootDir, "git", "submodule", "absorbgitdirs", "--", submodulePath))
}

func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
Expand Down
191 changes: 191 additions & 0 deletions eng/_util/cmd/submodule-refresh/submodule-refresh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
Expand Down Expand Up @@ -130,3 +131,193 @@ func TestRefreshRetriesOnlyInit(t *testing.T) {
})
}
}

func TestRefreshRef(t *testing.T) {
if _, err := exec.LookPath("git"); err != nil {
t.Skip("Git is not installed")
}
oldRef, oldOrigin := *ref, *origin
oldShallow, oldSkipPatch := *shallow, *skipPatch
t.Cleanup(func() {
*ref, *origin = oldRef, oldOrigin
*shallow, *skipPatch = oldShallow, oldSkipPatch
})
t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
t.Setenv("GIT_ALLOW_PROTOCOL", "file")
t.Setenv("GIT_AUTHOR_NAME", "Test")
t.Setenv("GIT_AUTHOR_EMAIL", "test@example.com")
t.Setenv("GIT_COMMITTER_NAME", "Test")
t.Setenv("GIT_COMMITTER_EMAIL", "test@example.com")

git := func(dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
write := func(path, contents string) {
t.Helper()
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}
source := filepath.Join(t.TempDir(), "source with spaces")
git(t.TempDir(), "init", source)
git(source, "config", "core.autocrlf", "false")
write(filepath.Join(source, "file.txt"), "original\n")
git(source, "add", ".")
git(source, "commit", "-m", "Pinned revision")
pinned := git(source, "rev-parse", "HEAD")
write(filepath.Join(source, "upstream.txt"), "upstream change\n")
git(source, "add", ".")
git(source, "commit", "-m", "Override revision")
override := git(source, "rev-parse", "HEAD")
gerritRef := "refs/changes/34/123434/2"
git(source, "update-ref", gerritRef, override)

for _, tt := range []struct {
name string
existing bool
gerrit bool
defaultOrigin bool
full bool
skipPatch bool
missing bool
}{
{name: "fresh SHA"},
{name: "fresh Gerrit ref", gerrit: true},
{name: "existing SHA", existing: true},
{name: "existing Gerrit ref", existing: true, gerrit: true},
{name: "default origin", defaultOrigin: true},
{name: "full fetch", full: true},
{name: "skip patches", skipPatch: true},
{name: "fresh missing ref", missing: true},
{name: "existing missing ref", existing: true, missing: true},
} {
t.Run(tt.name, func(t *testing.T) {
root := t.TempDir()
sub := filepath.Join(root, "go")
git(root, "init")
git(root, "config", "-f", ".gitmodules", "submodule.go.path", "go")
git(root, "config", "-f", ".gitmodules", "submodule.go.url", source)
git(root, "add", ".gitmodules")
git(root, "update-index", "--add", "--cacheinfo", "160000", pinned, "go")
git(root, "commit", "-m", "Pin submodule")
write(filepath.Join(root, ".git-go-patch"), `{"SubmoduleDir":"go","PatchesDir":"patches"}`)
if err := os.Mkdir(filepath.Join(root, "patches"), 0o700); err != nil {
t.Fatal(err)
}
write(filepath.Join(root, "patches", "0001-change.patch"),
"diff --git a/file.txt b/file.txt\n--- a/file.txt\n+++ b/file.txt\n@@ -1 +1 @@\n-original\n+patched\n")
existingOrigin := filepath.Join(root, "unavailable-origin")
if tt.existing {
git(root, "submodule", "update", "--init")
git(sub, "remote", "set-url", "origin", existingOrigin)
write(filepath.Join(sub, "file.txt"), "local change\n")
write(filepath.Join(sub, "untracked.txt"), "local file\n")
}
if !tt.defaultOrigin {
// Explicit -origin must not fetch from the pinned submodule source.
git(root, "config", "-f", ".gitmodules", "submodule.go.url", existingOrigin)
git(root, "config", "submodule.go.url", existingOrigin)
}
*ref, *origin = override, source
*shallow, *skipPatch = !tt.full, tt.skipPatch
if tt.gerrit {
*ref = gerritRef
}
if tt.defaultOrigin {
*origin = ""
}
if tt.missing {
*ref = "refs/changes/missing"
}
trace := filepath.Join(root, "git.trace")
t.Setenv("GIT_TRACE", filepath.ToSlash(trace))
err := refresh(root)
if (err != nil) != tt.missing {
t.Fatalf("refresh error = %v, want error: %v", err, tt.missing)
}
wantCommit := override
if tt.missing {
wantCommit = pinned
}
if got := git(root, "rev-parse", ":go"); got != wantCommit {
t.Fatalf("staged revision = %s, want %s", got, wantCommit)
}
checkContent := func(want string) {
t.Helper()
got, err := os.ReadFile(filepath.Join(sub, "file.txt"))
if err != nil || string(got) != want {
t.Fatalf("file.txt = %q, %v; want %q", got, err, want)
}
}
if tt.missing {
if tt.existing {
checkContent("local change\n")
if got := git(sub, "rev-parse", "HEAD"); got != pinned {
t.Fatalf("failed fetch changed HEAD to %s", got)
}
}
} else {
if got := git(sub, "rev-parse", "HEAD"); got != override {
t.Fatalf("HEAD = %s, want %s", got, override)
}
wantOrigin := source
if tt.existing {
wantOrigin = existingOrigin
}
if got := git(sub, "remote", "get-url", "origin"); got != wantOrigin {
t.Fatalf("origin = %q, want %q", got, wantOrigin)
}
if !tt.existing {
if got := git(sub, "rev-parse", "--is-shallow-repository"); got != strconv.FormatBool(!tt.full) {
t.Fatalf("unexpected shallow state: %s", got)
}
}
if tt.skipPatch {
checkContent("original\n")
} else {
checkContent("patched\n")
}
if _, err := os.Stat(filepath.Join(sub, "untracked.txt")); !os.IsNotExist(err) {
t.Fatalf("untracked file not cleaned: %v", err)
}
*ref, *origin, *skipPatch = "", "", false
if err := refresh(root); err != nil {
t.Fatal(err)
}
checkContent("patched\n")
if got := git(sub, "rev-parse", "HEAD"); got != override {
t.Fatalf("subsequent refresh changed HEAD to %s", got)
}
}
traceData, err := os.ReadFile(trace)
if err != nil {
t.Fatal(err)
}
wantFetches := 1
if tt.missing {
wantFetches = 3
}
if got := strings.Count(string(traceData), "built-in: git fetch "); got != wantFetches {
t.Fatalf("fetch count = %d, want %d\n%s", got, wantFetches, traceData)
}
if !tt.missing && !tt.existing {
git(root, "restore", "--staged", "--", "go")
if err := refresh(root); err != nil {
t.Fatalf("restore pinned revision: %v", err)
}
if got := git(sub, "rev-parse", "HEAD"); got != pinned {
t.Fatalf("restored HEAD = %s, want %s", got, pinned)
}
checkContent("patched\n")
}
})
}
}
17 changes: 17 additions & 0 deletions eng/doc/DeveloperGuide.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,23 @@ go version
go test -short ./...
```

## Testing an upstream Go change

Manually run a [PR or rolling pipeline](../pipeline/README.md) using **Run pipeline** in Azure DevOps.
Select the outer repository branch (or ref under **Branch/tag**) to test, then set `submoduleRef` to an upstream commit SHA or Gerrit ref, such as `refs/changes/34/123434/2`.
Comment thread
qmuntal marked this conversation as resolved.
Resolved
It's fine to select someone else's branch and submodule ref.
Leave it as `nil` to use the recorded submodule commit.
The override applies to both innerloop and outerloop pipelines.

Overrides are fetched anonymously from `https://go.googlesource.com/go`.
Public PR pipelines also accept `submoduleRepository`, for example an HTTPS GitHub fork URL, to test a commit that has not been uploaded to Gerrit.
This parameter is only used when `submoduleRef` is set; internal rolling pipelines always fetch overrides from the public Go repository, without the internal mirror credentials.

Patches are applied onto the requested commit, and patch conflicts fail the build as usual.
Fresh-agent test retries use the same resolved commit as the first attempt, even if the ref has advanced e.g. on Gerrit.

To use an override locally, see the [`submodule-refresh` options](../README.md#patch-files).

## IDE setup

### VS Code
Expand Down
12 changes: 11 additions & 1 deletion eng/pipeline/pr-outerloop-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ parameters:
values:
- "\U0001F535 pr-outerloop-pipeline.yml \U0001F535 \U0001F535"
default: "\U0001F535 pr-outerloop-pipeline.yml \U0001F535 \U0001F535"
- name: submoduleRef
displayName: 'Go submodule commit or ref override (nil uses the pinned commit)'
type: string
default: 'nil'
- name: submoduleRepository
displayName: 'Go submodule source repository (used only with a ref override)'
type: string
default: 'https://go.googlesource.com/go'
variables:
- template: variables/pipeline.yml
resources:
Expand All @@ -41,5 +49,7 @@ resources:
stages:
- template: stages/go-builder-matrix-stages.yml
parameters:
ctx: {}
ctx:
GoSubmoduleRef: ${{ parameters.submoduleRef }}
GoSubmoduleRepository: ${{ parameters.submoduleRepository }}
outerloop: true
12 changes: 11 additions & 1 deletion eng/pipeline/pr-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ parameters:
values:
- "\U0001F535 pr-pipeline.yml \U0001F535 \U0001F535"
default: "\U0001F535 pr-pipeline.yml \U0001F535 \U0001F535"
- name: submoduleRef
displayName: 'Go submodule commit or ref override (nil uses the pinned commit)'
type: string
default: 'nil'
- name: submoduleRepository
displayName: 'Go submodule source repository (used only with a ref override)'
type: string
default: 'https://go.googlesource.com/go'
variables:
- template: variables/pipeline.yml
resources:
Expand All @@ -41,6 +49,8 @@ resources:
stages:
- template: stages/go-builder-matrix-stages.yml
parameters:
ctx: {}
ctx:
GoSubmoduleRef: ${{ parameters.submoduleRef }}
GoSubmoduleRepository: ${{ parameters.submoduleRepository }}
innerloop: true
buildandpack: true
Loading
Loading