Skip to content

Commit fc92e6a

Browse files
authored
Merge pull request #2690 from microsoft/copilot/add-ci-submodule-commit-option
Allow CI to test alternate Go submodule revisions
2 parents 1b546ad + bedafa2 commit fc92e6a

12 files changed

Lines changed: 398 additions & 30 deletions

‎eng/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,14 @@ installing `git-go-patch`:
8181
* `pwsh eng/run.ps1 submodule-refresh` updates the submodule and applies the
8282
patches.
8383
* Pass `-commits` to apply each patch as a separate commit.
84+
* Pass `-ref <commit-or-ref>` to fetch and use a different upstream revision.
85+
Add `-origin <repository-url>` to fetch it from another repository, or
86+
`-shallow` to limit the fetch to depth 1.
87+
This stages the selected `go` submodule commit in the outer repository so
88+
subsequent refreshes keep using it. To return to the committed revision,
89+
run `git restore --staged -- go` from the repository root, then refresh again.
90+
Refreshing discards changes in the submodule; don't commit the staged override
91+
unless you intend to update the recorded submodule commit.
8492
* `pwsh eng/run.ps1 build -refresh` refreshes the submodule and applies patches
8593
and then goes on to build the Microsoft build of Go.
8694

‎eng/_util/cmd/submodule-refresh/submodule-refresh.go‎

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ import (
1111
"io"
1212
"log"
1313
"os"
14+
"os/exec"
1415
"path/filepath"
1516

17+
"github.com/microsoft/go-infra/executil"
1618
"github.com/microsoft/go-infra/patch"
1719
"github.com/microsoft/go-infra/submodule"
1820
"github.com/microsoft/go/_util/buildutil"
@@ -28,6 +30,7 @@ var (
2830
skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.")
2931
take = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.")
3032
origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
33+
ref = flag.String("ref", "", "Fetch this commit or ref instead of the recorded submodule commit, and stage it in the outer repository so subsequent refreshes use it rather than restoring the original commit, desirable in some CI scenarios.")
3134
shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
3235
fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
3336
)
@@ -58,14 +61,18 @@ func main() {
5861
}
5962

6063
func refresh(rootDir string) error {
61-
if err := buildutil.Retry(3, func() error {
62-
return submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow)
63-
}); err != nil {
64+
config, err := patch.FindAncestorConfig(rootDir)
65+
if err != nil {
6466
return err
6567
}
6668

67-
config, err := patch.FindAncestorConfig(rootDir)
68-
if err != nil {
69+
if *ref != "" {
70+
if err := initRef(config.RootDir, config.SubmoduleDir); err != nil {
71+
return err
72+
}
73+
} else if err := buildutil.Retry(3, func() error {
74+
return submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow)
75+
}); err != nil {
6976
return err
7077
}
7178

@@ -117,6 +124,63 @@ func refresh(rootDir string) error {
117124
return nil
118125
}
119126

127+
func initRef(rootDir, submodulePath string) error {
128+
// Register the submodule without fetching the recorded submodule commit.
129+
if err := executil.Run(executil.Dir(rootDir, "git", "submodule", "init", "--", submodulePath)); err != nil {
130+
return err
131+
}
132+
source := *origin
133+
if source == "" {
134+
var err error
135+
source, err = executil.SpaceTrimmedCombinedOutput(executil.Dir(rootDir, "git", "config", "--get", "submodule.go.url"))
136+
if err != nil {
137+
return err
138+
}
139+
}
140+
submoduleDir := filepath.Join(rootDir, submodulePath)
141+
if err := executil.Run(executil.Dir(rootDir, "git", "init", "--", submoduleDir)); err != nil {
142+
return err
143+
}
144+
// A later refresh may need to fetch the recorded submodule commit from this remote.
145+
if err := executil.RunQuiet(executil.Dir(submoduleDir, "git", "config", "--get", "remote.origin.url")); err != nil {
146+
var exitErr *exec.ExitError
147+
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
148+
return err
149+
}
150+
if err := executil.Run(executil.Dir(submoduleDir, "git", "remote", "add", "origin", source)); err != nil {
151+
return err
152+
}
153+
}
154+
args := []string{}
155+
if *fetchBearerToken != "" {
156+
args = append(args, "-c", "http.extraheader=AUTHORIZATION: bearer "+*fetchBearerToken)
157+
}
158+
args = append(args, "fetch", "--no-tags")
159+
if *shallow {
160+
args = append(args, "--depth=1")
161+
}
162+
args = append(args, "--", source, *ref)
163+
if err := buildutil.Retry(3, func() error {
164+
return executil.Run(executil.Dir(submoduleDir, "git", args...))
165+
}); err != nil {
166+
return err
167+
}
168+
commit, err := executil.SpaceTrimmedCombinedOutput(executil.Dir(submoduleDir, "git", "rev-parse", "--verify", "FETCH_HEAD^{commit}"))
169+
if err != nil {
170+
return err
171+
}
172+
if err := executil.Run(executil.Dir(submoduleDir, "git", "checkout", "--detach", "--force", commit)); err != nil {
173+
return err
174+
}
175+
// Reset and later refreshes run "git submodule update", which checks out the
176+
// commit recorded in the outer index. Stage the override so they don't
177+
// restore the original commit before applying patches.
178+
if err := executil.Run(executil.Dir(rootDir, "git", "update-index", "--cacheinfo", "160000", commit, submodulePath)); err != nil {
179+
return err
180+
}
181+
return executil.Run(executil.Dir(rootDir, "git", "submodule", "absorbgitdirs", "--", submodulePath))
182+
}
183+
120184
func copyFile(src, dst string) error {
121185
in, err := os.Open(src)
122186
if err != nil {

‎eng/_util/cmd/submodule-refresh/submodule-refresh_test.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package main
77
import (
88
"fmt"
99
"os"
10+
"os/exec"
1011
"path/filepath"
1112
"runtime"
1213
"strconv"
@@ -151,3 +152,193 @@ func TestRefreshRetriesOnlyInit(t *testing.T) {
151152
})
152153
}
153154
}
155+
156+
func TestRefreshRef(t *testing.T) {
157+
if _, err := exec.LookPath("git"); err != nil {
158+
t.Skip("Git is not installed")
159+
}
160+
oldRef, oldOrigin := *ref, *origin
161+
oldShallow, oldSkipPatch := *shallow, *skipPatch
162+
t.Cleanup(func() {
163+
*ref, *origin = oldRef, oldOrigin
164+
*shallow, *skipPatch = oldShallow, oldSkipPatch
165+
})
166+
t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
167+
t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
168+
t.Setenv("GIT_ALLOW_PROTOCOL", "file")
169+
t.Setenv("GIT_AUTHOR_NAME", "Test")
170+
t.Setenv("GIT_AUTHOR_EMAIL", "test@example.com")
171+
t.Setenv("GIT_COMMITTER_NAME", "Test")
172+
t.Setenv("GIT_COMMITTER_EMAIL", "test@example.com")
173+
174+
git := func(dir string, args ...string) string {
175+
t.Helper()
176+
cmd := exec.Command("git", args...)
177+
cmd.Dir = dir
178+
out, err := cmd.CombinedOutput()
179+
if err != nil {
180+
t.Fatalf("git %v: %v\n%s", args, err, out)
181+
}
182+
return strings.TrimSpace(string(out))
183+
}
184+
write := func(path, contents string) {
185+
t.Helper()
186+
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
187+
t.Fatal(err)
188+
}
189+
}
190+
source := filepath.Join(t.TempDir(), "source with spaces")
191+
git(t.TempDir(), "init", source)
192+
git(source, "config", "core.autocrlf", "false")
193+
write(filepath.Join(source, "file.txt"), "original\n")
194+
git(source, "add", ".")
195+
git(source, "commit", "-m", "Pinned revision")
196+
pinned := git(source, "rev-parse", "HEAD")
197+
write(filepath.Join(source, "upstream.txt"), "upstream change\n")
198+
git(source, "add", ".")
199+
git(source, "commit", "-m", "Override revision")
200+
override := git(source, "rev-parse", "HEAD")
201+
gerritRef := "refs/changes/34/123434/2"
202+
git(source, "update-ref", gerritRef, override)
203+
204+
for _, tt := range []struct {
205+
name string
206+
existing bool
207+
gerrit bool
208+
defaultOrigin bool
209+
full bool
210+
skipPatch bool
211+
missing bool
212+
}{
213+
{name: "fresh SHA"},
214+
{name: "fresh Gerrit ref", gerrit: true},
215+
{name: "existing SHA", existing: true},
216+
{name: "existing Gerrit ref", existing: true, gerrit: true},
217+
{name: "default origin", defaultOrigin: true},
218+
{name: "full fetch", full: true},
219+
{name: "skip patches", skipPatch: true},
220+
{name: "fresh missing ref", missing: true},
221+
{name: "existing missing ref", existing: true, missing: true},
222+
} {
223+
t.Run(tt.name, func(t *testing.T) {
224+
root := t.TempDir()
225+
sub := filepath.Join(root, "go")
226+
git(root, "init")
227+
git(root, "config", "-f", ".gitmodules", "submodule.go.path", "go")
228+
git(root, "config", "-f", ".gitmodules", "submodule.go.url", source)
229+
git(root, "add", ".gitmodules")
230+
git(root, "update-index", "--add", "--cacheinfo", "160000", pinned, "go")
231+
git(root, "commit", "-m", "Pin submodule")
232+
write(filepath.Join(root, ".git-go-patch"), `{"SubmoduleDir":"go","PatchesDir":"patches"}`)
233+
if err := os.Mkdir(filepath.Join(root, "patches"), 0o700); err != nil {
234+
t.Fatal(err)
235+
}
236+
write(filepath.Join(root, "patches", "0001-change.patch"),
237+
"diff --git a/file.txt b/file.txt\n--- a/file.txt\n+++ b/file.txt\n@@ -1 +1 @@\n-original\n+patched\n")
238+
existingOrigin := filepath.Join(root, "unavailable-origin")
239+
if tt.existing {
240+
git(root, "submodule", "update", "--init")
241+
git(sub, "remote", "set-url", "origin", existingOrigin)
242+
write(filepath.Join(sub, "file.txt"), "local change\n")
243+
write(filepath.Join(sub, "untracked.txt"), "local file\n")
244+
}
245+
if !tt.defaultOrigin {
246+
// Explicit -origin must not fetch from the pinned submodule source.
247+
git(root, "config", "-f", ".gitmodules", "submodule.go.url", existingOrigin)
248+
git(root, "config", "submodule.go.url", existingOrigin)
249+
}
250+
*ref, *origin = override, source
251+
*shallow, *skipPatch = !tt.full, tt.skipPatch
252+
if tt.gerrit {
253+
*ref = gerritRef
254+
}
255+
if tt.defaultOrigin {
256+
*origin = ""
257+
}
258+
if tt.missing {
259+
*ref = "refs/changes/missing"
260+
}
261+
trace := filepath.Join(root, "git.trace")
262+
t.Setenv("GIT_TRACE", filepath.ToSlash(trace))
263+
err := refresh(root)
264+
if (err != nil) != tt.missing {
265+
t.Fatalf("refresh error = %v, want error: %v", err, tt.missing)
266+
}
267+
wantCommit := override
268+
if tt.missing {
269+
wantCommit = pinned
270+
}
271+
if got := git(root, "rev-parse", ":go"); got != wantCommit {
272+
t.Fatalf("staged revision = %s, want %s", got, wantCommit)
273+
}
274+
checkContent := func(want string) {
275+
t.Helper()
276+
got, err := os.ReadFile(filepath.Join(sub, "file.txt"))
277+
if err != nil || string(got) != want {
278+
t.Fatalf("file.txt = %q, %v; want %q", got, err, want)
279+
}
280+
}
281+
if tt.missing {
282+
if tt.existing {
283+
checkContent("local change\n")
284+
if got := git(sub, "rev-parse", "HEAD"); got != pinned {
285+
t.Fatalf("failed fetch changed HEAD to %s", got)
286+
}
287+
}
288+
} else {
289+
if got := git(sub, "rev-parse", "HEAD"); got != override {
290+
t.Fatalf("HEAD = %s, want %s", got, override)
291+
}
292+
wantOrigin := source
293+
if tt.existing {
294+
wantOrigin = existingOrigin
295+
}
296+
if got := git(sub, "remote", "get-url", "origin"); got != wantOrigin {
297+
t.Fatalf("origin = %q, want %q", got, wantOrigin)
298+
}
299+
if !tt.existing {
300+
if got := git(sub, "rev-parse", "--is-shallow-repository"); got != strconv.FormatBool(!tt.full) {
301+
t.Fatalf("unexpected shallow state: %s", got)
302+
}
303+
}
304+
if tt.skipPatch {
305+
checkContent("original\n")
306+
} else {
307+
checkContent("patched\n")
308+
}
309+
if _, err := os.Stat(filepath.Join(sub, "untracked.txt")); !os.IsNotExist(err) {
310+
t.Fatalf("untracked file not cleaned: %v", err)
311+
}
312+
*ref, *origin, *skipPatch = "", "", false
313+
if err := refresh(root); err != nil {
314+
t.Fatal(err)
315+
}
316+
checkContent("patched\n")
317+
if got := git(sub, "rev-parse", "HEAD"); got != override {
318+
t.Fatalf("subsequent refresh changed HEAD to %s", got)
319+
}
320+
}
321+
traceData, err := os.ReadFile(trace)
322+
if err != nil {
323+
t.Fatal(err)
324+
}
325+
wantFetches := 1
326+
if tt.missing {
327+
wantFetches = 3
328+
}
329+
if got := strings.Count(string(traceData), "built-in: git fetch "); got != wantFetches {
330+
t.Fatalf("fetch count = %d, want %d\n%s", got, wantFetches, traceData)
331+
}
332+
if !tt.missing && !tt.existing {
333+
git(root, "restore", "--staged", "--", "go")
334+
if err := refresh(root); err != nil {
335+
t.Fatalf("restore pinned revision: %v", err)
336+
}
337+
if got := git(sub, "rev-parse", "HEAD"); got != pinned {
338+
t.Fatalf("restored HEAD = %s, want %s", got, pinned)
339+
}
340+
checkContent("patched\n")
341+
}
342+
})
343+
}
344+
}

‎eng/doc/DeveloperGuide.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,23 @@ go version
117117
go test -short ./...
118118
```
119119

120+
## Testing an upstream Go change
121+
122+
Manually run a [PR or rolling pipeline](../pipeline/README.md) using **Run pipeline** in Azure DevOps.
123+
Select the outer repository branch (or ref under **Branch/tag**) to test, then set `submoduleRef` to an upstream commit SHA or Gerrit ref, such as `refs/changes/34/123434/2`.
124+
It's fine to select someone else's branch and submodule ref.
125+
Leave it as `nil` to use the recorded submodule commit.
126+
The override applies to both innerloop and outerloop pipelines.
127+
128+
Overrides are fetched anonymously from `https://go.googlesource.com/go`.
129+
Public PR pipelines also accept `submoduleRepository`, for example an HTTPS GitHub fork URL, to test a commit that has not been uploaded to Gerrit.
130+
This parameter is only used when `submoduleRef` is set; internal rolling pipelines always fetch overrides from the public Go repository, without the internal mirror credentials.
131+
132+
Patches are applied onto the requested commit, and patch conflicts fail the build as usual.
133+
Fresh-agent test retries use the same resolved commit as the first attempt, even if the ref has advanced e.g. on Gerrit.
134+
135+
To use an override locally, see the [`submodule-refresh` options](../README.md#patch-files).
136+
120137
## IDE setup
121138

122139
### VS Code

‎eng/pipeline/pr-outerloop-pipeline.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ parameters:
2424
values:
2525
- "\U0001F535 pr-outerloop-pipeline.yml \U0001F535 \U0001F535"
2626
default: "\U0001F535 pr-outerloop-pipeline.yml \U0001F535 \U0001F535"
27+
- name: submoduleRef
28+
displayName: 'Go submodule commit or ref override (nil uses the pinned commit)'
29+
type: string
30+
default: 'nil'
31+
- name: submoduleRepository
32+
displayName: 'Go submodule source repository (used only with a ref override)'
33+
type: string
34+
default: 'https://go.googlesource.com/go'
2735
variables:
2836
- template: variables/pipeline.yml
2937
resources:
@@ -43,5 +51,7 @@ resources:
4351
stages:
4452
- template: stages/go-builder-matrix-stages.yml
4553
parameters:
46-
ctx: {}
54+
ctx:
55+
GoSubmoduleRef: ${{ parameters.submoduleRef }}
56+
GoSubmoduleRepository: ${{ parameters.submoduleRepository }}
4757
outerloop: true

‎eng/pipeline/pr-pipeline.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ parameters:
2424
values:
2525
- "\U0001F535 pr-pipeline.yml \U0001F535 \U0001F535"
2626
default: "\U0001F535 pr-pipeline.yml \U0001F535 \U0001F535"
27+
- name: submoduleRef
28+
displayName: 'Go submodule commit or ref override (nil uses the pinned commit)'
29+
type: string
30+
default: 'nil'
31+
- name: submoduleRepository
32+
displayName: 'Go submodule source repository (used only with a ref override)'
33+
type: string
34+
default: 'https://go.googlesource.com/go'
2735
variables:
2836
- template: variables/pipeline.yml
2937
resources:
@@ -43,6 +51,8 @@ resources:
4351
stages:
4452
- template: stages/go-builder-matrix-stages.yml
4553
parameters:
46-
ctx: {}
54+
ctx:
55+
GoSubmoduleRef: ${{ parameters.submoduleRef }}
56+
GoSubmoduleRepository: ${{ parameters.submoduleRepository }}
4757
innerloop: true
4858
buildandpack: true

0 commit comments

Comments
 (0)