From 7566e54383d4cf650c50922949939d06d73e1345 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 20 Mar 2026 09:59:04 -0700 Subject: [PATCH 1/5] Parallelize Patch Build sequence test --- .github/workflows/patch-build.yml | 50 ++++++----- .../submodule-refresh/submodule-refresh.go | 47 +++++++++++ .../write-patch-matrix/write-patch-matrix.go | 83 +++++++++++++++++++ 3 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 eng/_util/cmd/write-patch-matrix/write-patch-matrix.go diff --git a/.github/workflows/patch-build.yml b/.github/workflows/patch-build.yml index af26c3217f8..1ba5a1aadaa 100644 --- a/.github/workflows/patch-build.yml +++ b/.github/workflows/patch-build.yml @@ -16,37 +16,47 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + list_patches: + name: Generate patch build matrix + runs-on: ubuntu-latest + outputs: + patches: ${{ steps.list.outputs.patches }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Write patch matrix + id: list + run: pwsh eng/run.ps1 write-patch-matrix -github-actions + build_patches: - name: Patches Build in Order + name: "Build up to ${{ matrix.patch.name }}" + needs: list_patches runs-on: ubuntu-latest + strategy: + matrix: + patch: ${{ fromJson(needs.list_patches.outputs.patches) }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version: '1.25' - - name: Set mock git config name/email run: | git config --global user.email "joe@blogs.com" git config --global user.name "Joe Blogs" - - name: Build patches + - name: Apply patches 1-${{ matrix.patch.number }} + run: pwsh eng/run.ps1 submodule-refresh -commits -take ${{ matrix.patch.number }} + + - name: Build run: | - for file in $(ls -v patches/*.patch); do - echo "::group::Building $file" - cd ${{ github.workspace }}/go - git am --whitespace=nowarn ${{ github.workspace }}/$file - cd ${{ github.workspace }}/go/src - bash make.bash - ${{ github.workspace }}/go/bin/go mod vendor - cd ${{ github.workspace }}/go/src/cmd - ${{ github.workspace }}/go/bin/go mod vendor - cd ${{ github.workspace }}/go/src - # Check if the vendor directory is clean - git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1) - echo "::endgroup::" - done + cd ${{ github.workspace }}/go/src + pwsh eng/run.ps1 build + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src/cmd + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src + # Check if the vendor directory is clean + git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1) diff --git a/eng/_util/cmd/submodule-refresh/submodule-refresh.go b/eng/_util/cmd/submodule-refresh/submodule-refresh.go index effc1f99b22..8062c028f21 100644 --- a/eng/_util/cmd/submodule-refresh/submodule-refresh.go +++ b/eng/_util/cmd/submodule-refresh/submodule-refresh.go @@ -5,10 +5,13 @@ package main import ( + "errors" "flag" "fmt" + "io" "os" "path/filepath" + "slices" "github.com/microsoft/go-infra/patch" "github.com/microsoft/go-infra/submodule" @@ -22,6 +25,7 @@ applies patches to the stage by default, or optionally as commits. var ( commits = flag.Bool("commits", false, "Apply the patches as commits.") skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.") + take = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.") origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.") shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.") fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.") @@ -75,8 +79,51 @@ func refresh(rootDir string) error { mode = patch.ApplyModeCommits } + if *take >= 0 { + // The patch API applies all patches in a directory. To apply only the + // first N, copy them into a temporary directory and point the config there. + patchesDir := filepath.Join(config.RootDir, config.PatchesDir) + matches, err := filepath.Glob(filepath.Join(patchesDir, "*.patch")) + if err != nil { + return err + } + slices.Sort(matches) + if *take > len(matches) { + return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, len(matches)) + } + tmpDir, err := os.MkdirTemp("", "patches-take-*") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + for _, src := range matches[:*take] { + if err := copyFile(src, filepath.Join(tmpDir, filepath.Base(src))); err != nil { + return err + } + } + relTmpDir, err := filepath.Rel(config.RootDir, tmpDir) + if err != nil { + return err + } + config.PatchesDir = relTmpDir + } + if err := patch.Apply(config, mode); err != nil { return err } return nil } + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, out.Close()) +} diff --git a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go new file mode 100644 index 00000000000..226f0937fbb --- /dev/null +++ b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/microsoft/go-infra/patch" +) + +const description = ` +This command reads patch files from the patches/ directory and writes a JSON +array to stdout suitable for use as a GitHub Actions matrix. Each entry contains +a "number" (1-based position) and "name" (filename). The patches are sorted in +name order. + +Use -github-actions to write the result to $GITHUB_OUTPUT as "patches=". +` + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + ghActions := flag.Bool("github-actions", false, "Write the result to $GITHUB_OUTPUT.") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + flag.Parse() + + type entry struct { + Number int `json:"number"` + Name string `json:"name"` + } + var entries []entry + + config, err := patch.FindAncestorConfig(".") + if err != nil { + return err + } + + n := 1 + if err := patch.WalkGoPatches(config, func(s string) error { + entries = append(entries, entry{Number: n, Name: filepath.Base(s)}) + n++ + return nil + }); err != nil { + return fmt.Errorf("error walking patches: %v", err) + } + + out, err := json.Marshal(entries) + if err != nil { + return err + } + + if *ghActions { + ghOutputPath := os.Getenv("GITHUB_OUTPUT") + if ghOutputPath == "" { + return fmt.Errorf("GITHUB_OUTPUT environment variable is not set") + } + f, err := os.OpenFile(ghOutputPath, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + return err + } + _, writeErr := fmt.Fprintf(f, "patches=%s\n", out) + return errors.Join(writeErr, f.Close()) + } else { + fmt.Println(string(out)) + } + return nil +} From 97324d95d9e56901101fc7024aec57ade2ab6900 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 20 Mar 2026 11:08:42 -0700 Subject: [PATCH 2/5] Apply feedback Use eng/artifacts. Rely on WalkGoPatches. Explicitly don't support zero patch files. --- .../submodule-refresh/submodule-refresh.go | 36 +++++++++---------- .../write-patch-matrix/write-patch-matrix.go | 7 ++-- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/eng/_util/cmd/submodule-refresh/submodule-refresh.go b/eng/_util/cmd/submodule-refresh/submodule-refresh.go index 8062c028f21..a4c62bbb947 100644 --- a/eng/_util/cmd/submodule-refresh/submodule-refresh.go +++ b/eng/_util/cmd/submodule-refresh/submodule-refresh.go @@ -9,9 +9,9 @@ import ( "flag" "fmt" "io" + "log" "os" "path/filepath" - "slices" "github.com/microsoft/go-infra/patch" "github.com/microsoft/go-infra/submodule" @@ -82,30 +82,30 @@ func refresh(rootDir string) error { if *take >= 0 { // The patch API applies all patches in a directory. To apply only the // first N, copy them into a temporary directory and point the config there. - patchesDir := filepath.Join(config.RootDir, config.PatchesDir) - matches, err := filepath.Glob(filepath.Join(patchesDir, "*.patch")) - if err != nil { + tmpDirRelative := filepath.Join("eng", "artifacts", "submodule-refresh", "patch-subset") + tmpDir := filepath.Join(config.RootDir, tmpDirRelative) + if err := os.RemoveAll(tmpDir); err != nil { return err } - slices.Sort(matches) - if *take > len(matches) { - return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, len(matches)) - } - tmpDir, err := os.MkdirTemp("", "patches-take-*") - if err != nil { + if err := os.MkdirAll(tmpDir, 0o777); err != nil { return err } - defer os.RemoveAll(tmpDir) - for _, src := range matches[:*take] { - if err := copyFile(src, filepath.Join(tmpDir, filepath.Base(src))); err != nil { - return err + i := 0 + if err := patch.WalkGoPatches(config, func(path string) error { + if i >= *take { + log.Printf("Not including patch %q\n", path) + return nil } - } - relTmpDir, err := filepath.Rel(config.RootDir, tmpDir) - if err != nil { + i++ + log.Printf("Taking patch %q\n", path) + return copyFile(path, filepath.Join(tmpDir, filepath.Base(path))) + }); err != nil { return err } - config.PatchesDir = relTmpDir + if *take > i { + return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, i) + } + config.PatchesDir = tmpDirRelative } if err := patch.Apply(config, mode); err != nil { diff --git a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go index 226f0937fbb..49fa6ccf44c 100644 --- a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go +++ b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go @@ -19,8 +19,7 @@ import ( const description = ` This command reads patch files from the patches/ directory and writes a JSON array to stdout suitable for use as a GitHub Actions matrix. Each entry contains -a "number" (1-based position) and "name" (filename). The patches are sorted in -name order. +a "number" (1-based position) and "name" (filename). Use -github-actions to write the result to $GITHUB_OUTPUT as "patches=". ` @@ -60,6 +59,10 @@ func run() error { return fmt.Errorf("error walking patches: %v", err) } + if len(entries) == 0 { + return errors.New("no patches found") + } + out, err := json.Marshal(entries) if err != nil { return err From cad1dcbb0875c7162835734552881dcc5df3c0fa Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 20 Mar 2026 11:23:22 -0700 Subject: [PATCH 3/5] Fix Build directory --- .github/workflows/patch-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/patch-build.yml b/.github/workflows/patch-build.yml index 1ba5a1aadaa..4c947c0a949 100644 --- a/.github/workflows/patch-build.yml +++ b/.github/workflows/patch-build.yml @@ -52,8 +52,9 @@ jobs: - name: Build run: | - cd ${{ github.workspace }}/go/src + set -x pwsh eng/run.ps1 build + cd ${{ github.workspace }}/go/src ${{ github.workspace }}/go/bin/go mod vendor cd ${{ github.workspace }}/go/src/cmd ${{ github.workspace }}/go/bin/go mod vendor From 946f8012f0a34960a9a828096024320921e9825d Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 20 Mar 2026 11:26:22 -0700 Subject: [PATCH 4/5] For diag, make -github-actions additional, not exclusive --- eng/_util/cmd/write-patch-matrix/write-patch-matrix.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go index 49fa6ccf44c..20ed3c392ed 100644 --- a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go +++ b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go @@ -68,6 +68,8 @@ func run() error { return err } + fmt.Println(string(out)) + if *ghActions { ghOutputPath := os.Getenv("GITHUB_OUTPUT") if ghOutputPath == "" { @@ -79,8 +81,6 @@ func run() error { } _, writeErr := fmt.Fprintf(f, "patches=%s\n", out) return errors.Join(writeErr, f.Close()) - } else { - fmt.Println(string(out)) } return nil } From b175f6d083c7cb7efe26097ce8a54c8c97d563ac Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 20 Mar 2026 11:55:19 -0700 Subject: [PATCH 5/5] Skip the race build --- .github/workflows/patch-build.yml | 3 ++- eng/_util/cmd/build/build.go | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/patch-build.yml b/.github/workflows/patch-build.yml index 4c947c0a949..54e7c51c420 100644 --- a/.github/workflows/patch-build.yml +++ b/.github/workflows/patch-build.yml @@ -53,7 +53,8 @@ jobs: - name: Build run: | set -x - pwsh eng/run.ps1 build + # Don't build with the race detector. https://github.com/microsoft/go/issues/2204 + pwsh eng/run.ps1 build -skipbuildrace cd ${{ github.workspace }}/go/src ${{ github.workspace }}/go/bin/go mod vendor cd ${{ github.workspace }}/go/src/cmd diff --git a/eng/_util/cmd/build/build.go b/eng/_util/cmd/build/build.go index 7634bcc41f6..923d89e04ce 100644 --- a/eng/_util/cmd/build/build.go +++ b/eng/_util/cmd/build/build.go @@ -41,6 +41,7 @@ func main() { o := &options{} flag.BoolVar(&o.SkipBuild, "skipbuild", false, "Disable building Go.") + flag.BoolVar(&o.SkipBuildRace, "skipbuildrace", false, "Disable building Go with race detector.") flag.BoolVar(&o.Test, "test", false, "Enable running tests.") flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.") flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.") @@ -78,13 +79,14 @@ func main() { } type options struct { - SkipBuild bool - Test bool - PackBuild bool - PackSource bool - CreatePDB bool - Refresh bool - Experiment string + SkipBuild bool + SkipBuildRace bool + Test bool + PackBuild bool + PackSource bool + CreatePDB bool + Refresh bool + Experiment string TestJSONFlags *buildutil.TestJSONFlags @@ -202,7 +204,12 @@ func build(o *options) (err error) { // The race runtime requires cgo. // It isn't supported on arm or 386. // It's supported on arm64, but the official linux-arm64 distribution doesn't include it. - if os.Getenv("CGO_ENABLED") != "0" && targetArch != "arm" && targetArch != "arm64" && targetArch != "386" { + if !o.SkipBuildRace && + os.Getenv("CGO_ENABLED") != "0" && + targetArch != "arm" && + targetArch != "arm64" && + targetArch != "386" { + fmt.Println("---- Building race runtime...") err := runCommandLine( filepath.Join("..", "bin", "go"+executableExtension),