diff --git a/.github/workflows/patch-build.yml b/.github/workflows/patch-build.yml index af26c3217f8..54e7c51c420 100644 --- a/.github/workflows/patch-build.yml +++ b/.github/workflows/patch-build.yml @@ -16,37 +16,49 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + list_patches: + name: Generate patch build matrix + runs-on: ubuntu-latest + outputs: + patches: ${{ steps.list.outputs.patches }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Write patch matrix + id: list + run: pwsh eng/run.ps1 write-patch-matrix -github-actions + build_patches: - name: Patches Build in Order + name: "Build up to ${{ matrix.patch.name }}" + needs: list_patches runs-on: ubuntu-latest + strategy: + matrix: + patch: ${{ fromJson(needs.list_patches.outputs.patches) }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version: '1.25' - - name: Set mock git config name/email run: | git config --global user.email "joe@blogs.com" git config --global user.name "Joe Blogs" - - name: Build patches + - name: Apply patches 1-${{ matrix.patch.number }} + run: pwsh eng/run.ps1 submodule-refresh -commits -take ${{ matrix.patch.number }} + + - name: Build run: | - for file in $(ls -v patches/*.patch); do - echo "::group::Building $file" - cd ${{ github.workspace }}/go - git am --whitespace=nowarn ${{ github.workspace }}/$file - cd ${{ github.workspace }}/go/src - bash make.bash - ${{ github.workspace }}/go/bin/go mod vendor - cd ${{ github.workspace }}/go/src/cmd - ${{ github.workspace }}/go/bin/go mod vendor - cd ${{ github.workspace }}/go/src - # Check if the vendor directory is clean - git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1) - echo "::endgroup::" - done + set -x + # Don't build with the race detector. https://github.com/microsoft/go/issues/2204 + pwsh eng/run.ps1 build -skipbuildrace + cd ${{ github.workspace }}/go/src + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src/cmd + ${{ github.workspace }}/go/bin/go mod vendor + cd ${{ github.workspace }}/go/src + # Check if the vendor directory is clean + git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1) diff --git a/eng/_util/cmd/build/build.go b/eng/_util/cmd/build/build.go index 7634bcc41f6..923d89e04ce 100644 --- a/eng/_util/cmd/build/build.go +++ b/eng/_util/cmd/build/build.go @@ -41,6 +41,7 @@ func main() { o := &options{} flag.BoolVar(&o.SkipBuild, "skipbuild", false, "Disable building Go.") + flag.BoolVar(&o.SkipBuildRace, "skipbuildrace", false, "Disable building Go with race detector.") flag.BoolVar(&o.Test, "test", false, "Enable running tests.") flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.") flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.") @@ -78,13 +79,14 @@ func main() { } type options struct { - SkipBuild bool - Test bool - PackBuild bool - PackSource bool - CreatePDB bool - Refresh bool - Experiment string + SkipBuild bool + SkipBuildRace bool + Test bool + PackBuild bool + PackSource bool + CreatePDB bool + Refresh bool + Experiment string TestJSONFlags *buildutil.TestJSONFlags @@ -202,7 +204,12 @@ func build(o *options) (err error) { // The race runtime requires cgo. // It isn't supported on arm or 386. // It's supported on arm64, but the official linux-arm64 distribution doesn't include it. - if os.Getenv("CGO_ENABLED") != "0" && targetArch != "arm" && targetArch != "arm64" && targetArch != "386" { + if !o.SkipBuildRace && + os.Getenv("CGO_ENABLED") != "0" && + targetArch != "arm" && + targetArch != "arm64" && + targetArch != "386" { + fmt.Println("---- Building race runtime...") err := runCommandLine( filepath.Join("..", "bin", "go"+executableExtension), diff --git a/eng/_util/cmd/submodule-refresh/submodule-refresh.go b/eng/_util/cmd/submodule-refresh/submodule-refresh.go index effc1f99b22..a4c62bbb947 100644 --- a/eng/_util/cmd/submodule-refresh/submodule-refresh.go +++ b/eng/_util/cmd/submodule-refresh/submodule-refresh.go @@ -5,8 +5,11 @@ package main import ( + "errors" "flag" "fmt" + "io" + "log" "os" "path/filepath" @@ -22,6 +25,7 @@ applies patches to the stage by default, or optionally as commits. var ( commits = flag.Bool("commits", false, "Apply the patches as commits.") skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.") + take = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.") origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.") shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.") fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.") @@ -75,8 +79,51 @@ func refresh(rootDir string) error { mode = patch.ApplyModeCommits } + if *take >= 0 { + // The patch API applies all patches in a directory. To apply only the + // first N, copy them into a temporary directory and point the config there. + tmpDirRelative := filepath.Join("eng", "artifacts", "submodule-refresh", "patch-subset") + tmpDir := filepath.Join(config.RootDir, tmpDirRelative) + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + if err := os.MkdirAll(tmpDir, 0o777); err != nil { + return err + } + i := 0 + if err := patch.WalkGoPatches(config, func(path string) error { + if i >= *take { + log.Printf("Not including patch %q\n", path) + return nil + } + i++ + log.Printf("Taking patch %q\n", path) + return copyFile(path, filepath.Join(tmpDir, filepath.Base(path))) + }); err != nil { + return err + } + if *take > i { + return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, i) + } + config.PatchesDir = tmpDirRelative + } + if err := patch.Apply(config, mode); err != nil { return err } return nil } + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, out.Close()) +} diff --git a/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go new file mode 100644 index 00000000000..20ed3c392ed --- /dev/null +++ b/eng/_util/cmd/write-patch-matrix/write-patch-matrix.go @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/microsoft/go-infra/patch" +) + +const description = ` +This command reads patch files from the patches/ directory and writes a JSON +array to stdout suitable for use as a GitHub Actions matrix. Each entry contains +a "number" (1-based position) and "name" (filename). + +Use -github-actions to write the result to $GITHUB_OUTPUT as "patches=". +` + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + ghActions := flag.Bool("github-actions", false, "Write the result to $GITHUB_OUTPUT.") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + flag.Parse() + + type entry struct { + Number int `json:"number"` + Name string `json:"name"` + } + var entries []entry + + config, err := patch.FindAncestorConfig(".") + if err != nil { + return err + } + + n := 1 + if err := patch.WalkGoPatches(config, func(s string) error { + entries = append(entries, entry{Number: n, Name: filepath.Base(s)}) + n++ + return nil + }); err != nil { + return fmt.Errorf("error walking patches: %v", err) + } + + if len(entries) == 0 { + return errors.New("no patches found") + } + + out, err := json.Marshal(entries) + if err != nil { + return err + } + + fmt.Println(string(out)) + + if *ghActions { + ghOutputPath := os.Getenv("GITHUB_OUTPUT") + if ghOutputPath == "" { + return fmt.Errorf("GITHUB_OUTPUT environment variable is not set") + } + f, err := os.OpenFile(ghOutputPath, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + return err + } + _, writeErr := fmt.Fprintf(f, "patches=%s\n", out) + return errors.Join(writeErr, f.Close()) + } + return nil +}