Merge remote-tracking branch 'upstream/microsoft/main' into dev/auto-sync/microsoft/main

Author: gdams

.github/workflows/codeql.yml

@@ -23,16 +23,16 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6
+        uses: github/codeql-action/init@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6
+        uses: github/codeql-action/autobuild@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1
         with:
           working-directory: eng/_util
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6
+        uses: github/codeql-action/analyze@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1
         with:
           category: /language:go

.github/workflows/patch-build.yml

@@ -16,37 +16,48 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  list_patches:
+    name: Generate patch build matrix
+    runs-on: ubuntu-latest
+    outputs:
+      patches: ${{ steps.list.outputs.patches }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Write patch matrix
+        id: list
+        run: pwsh eng/run.ps1 write-patch-matrix -github-actions
+
   build_patches:
-    name: Patches Build in Order
+    name: "Build up to ${{ matrix.patch.name }}"
+    needs: list_patches
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        patch: ${{ fromJson(needs.list_patches.outputs.patches) }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           submodules: true
 
-      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
-        with:
-          go-version: '1.25'
-
       - name: Set mock git config name/email
         run: |
           git config --global user.email "joe@blogs.com"
           git config --global user.name "Joe Blogs"
 
-      - name: Build patches
+      - name: Apply patches 1-${{ matrix.patch.number }}
+        run: pwsh eng/run.ps1 submodule-refresh -commits -take ${{ matrix.patch.number }}
+
+      - name: Build
         run: |
-          for file in $(ls -v patches/*.patch); do
-            echo "::group::Building $file"
-            cd ${{ github.workspace }}/go
-            git am --whitespace=nowarn ${{ github.workspace }}/$file
-            cd ${{ github.workspace }}/go/src
-            bash make.bash
-            ${{ github.workspace }}/go/bin/go mod vendor
-            cd ${{ github.workspace }}/go/src/cmd
-            ${{ github.workspace }}/go/bin/go mod vendor
-            cd ${{ github.workspace }}/go/src
-            # Check if the vendor directory is clean
-            git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1)
-            echo "::endgroup::"
-          done
+          set -x
+          pwsh eng/run.ps1 build
+          cd ${{ github.workspace }}/go/src
+          ${{ github.workspace }}/go/bin/go mod vendor
+          cd ${{ github.workspace }}/go/src/cmd
+          ${{ github.workspace }}/go/bin/go mod vendor
+          cd ${{ github.workspace }}/go/src
+          # Check if the vendor directory is clean
+          git diff --exit-code vendor cmd/vendor || (echo "Vendor directories are not clean. Please run 'go mod vendor' in the appropriate directories and commit the changes." && exit 1)

.github/workflows/test.yml

@@ -11,7 +11,9 @@
 # time to hit the patch failure N times. However, the actual tests run in AzDO, so we can't
 # reasonably cancel them from here (GitHub Actions).
 
-name: "Test"
+# The "0:" prefix makes this workflow's checks sort above the alphabetical AzDO stage names on the
+# PR checks list, so patch failures are immediately visible without scrolling.
+name: "0: Test"
 
 on:
   pull_request:

README.md

@@ -57,95 +57,27 @@ There are a few additional support resources internal to Microsoft:
 
 ## Download and install
 
-We build the forked Go toolset with the following list of OS/Arch combinations:
-
-* `linux_amd64`
-* `linux_armv6l`
-* `linux_arm64`
-* `windows_amd64`
-* `windows_arm64`
-* `darwin_amd64`
-* `darwin_arm64`
-
-Visit the [Migration Guide](eng/doc/MigrationGuide.md) for guidance about how we
-recommend migrating existing Go projects to use the Microsoft build of Go. This
-guide also helps resolve commonly encountered issues.
-
-The following sections list the ways to get the Microsoft build of Go.
-
-> [!NOTE]
-> Don't see an option that works for you? Let us know!  
-> File a GitHub issue, or comment on an existing issue in this tag:
-  [![](https://img.shields.io/github/labels/microsoft/go/Area-Acquisition)](https://github.com/microsoft/go/labels/Area-Acquisition)
-
-### Docker Container Images
-
-**[microsoft/go-images](https://github.com/microsoft/go-images)** maintains and
-documents container images that are available on Microsoft Artifact Registry.
-
-### Azure Linux
-
-The **[Azure Linux](https://github.com/microsoft/azurelinux)** distribution
-includes the `golang` package, a build of this fork of Go.
-
-For more information about how to manage the `systemcrypto` migration from 1.24
-to 1.25 in Azure Linux 3, see
-[the `systemcrypto` section of the Migration Guide](eng/doc/MigrationGuide.md#migration-to-systemcrypto).
-
-### Ubuntu
-
-To install the Microsoft build of Go using an Ubuntu package, first set up the [Linux package repository for Microsoft Products](https://learn.microsoft.com/en-us/linux/packages).
-Packages are available in the Ubuntu 22.04 and 24.04 repositories.
-
-After the repository is added, install the Microsoft build of Go by running the following commands:
-
-```bash
-sudo apt-get update && sudo apt-get install msft-golang
-```
-
-### Azure Pipelines `GoTool@0` task
-
-The [`GoTool@0`](https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/go-tool-v0) Azure Pipelines build task supports installing the Microsoft build of Go.
-For more details, see [the GoTool@0 section of the Migration Guide](eng/doc/MigrationGuide.md#the-gotool0-azure-pipelines-task).
-
-### The `go-install.ps1` script
-
-The [cross-platform `go-install.ps1` script](https://github.com/microsoft/go-infra/tree/main/goinstallscript) installs the Microsoft build of Go.
-It can install specific versions or the latest releases.
-
-If you use Azure Pipelines, try running the script in a [script step](https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines) and pass the `-AzurePipelinePath` argument to automatically set up `go` in the environment for future steps.
-
-### Binary archive
-
-[Signed builds of Go](https://github.com/microsoft/go/blob/microsoft/main/eng/doc/Downloads.md)
-for several platforms are available as `zip` and `tar.gz` files.
-
-### Build from source
-
-#### Pre-patched source tarball
-
-[The microsoft/go GitHub releases](https://github.com/microsoft/go/releases)
-include a source tarball file ending in `.src.tar.gz`. After downloading and
-extracting the tar.gz file, build it using the
-[upstream instructions](https://go.dev/doc/install/source).
-
-> [!NOTE]
-> The `zip` file that GitHub offers for download on the [microsoft/go releases page](https://github.com/microsoft/go/releases) is incomplete: it doesn't include the `go` submodule.
-> Make sure to download the `.src.tar.gz` file instead, or [clone the repository using Git and set up the patched submodule](#clone-and-build).
-
-#### Clone and build
-
-First, clone this repository using Git and check out the desired tag or commit.
-
-If you want to contribute to the Microsoft build of Go project, read the [Developer Guide](eng/doc/DeveloperGuide.md).
-It lists the steps we recommend to set up a Microsoft build of Go development environment, execute your first build, run the standard library test suite, and contribute a PR.
-
-If you just want to build on your own machine, you may find it more
-convenient to use the tools provided by the `eng/run.ps1` script. We use this
-script for CI builds. See [eng/README.md](eng/README.md) for more details about
-`eng/run.ps1` and other repository infrastructure.
-
-Once built, the Microsoft build of Go binary is found at `go/bin/go`.
+We build the Microsoft build of Go toolset with the following OS/Arch combinations:
+
+| OS | `amd64` | `arm64` | `armv6l` |
+| --- | :---: | :---: | :---: |
+| `linux` | ✓ | ✓ | ✓ |
+| `windows` | ✓ | ✓ | |
+| `darwin` (macOS) | ✓ | ✓ | |
+
+Visit the [Migration Guide](eng/doc/MigrationGuide.md) for guidance about how we recommend migrating existing Go projects to use the Microsoft build of Go.
+This guide also helps resolve commonly encountered issues.
+
+The [Installation](eng/doc/Installation.md) documentation contains sections describing each of the following installation methods:
+
+* [Docker Container Images](eng/doc/Installation.md#docker-container-images)
+* [Azure Linux](eng/doc/Installation.md#azure-linux)
+* [Ubuntu](eng/doc/Installation.md#ubuntu)
+* [Azure Pipelines `GoTool@0` task](eng/doc/Installation.md#azure-pipelines-gotool0-task)
+* [GitHub Actions `setup-go` action](eng/doc/Installation.md#github-actions-setup-go-action)
+* [The `go-install.ps1` script](eng/doc/Installation.md#the-go-installps1-script)
+* [Binary archive](eng/doc/Installation.md#binary-archive)
+* [Build from source](eng/doc/Installation.md#build-from-source)
 
 ## Contributing
 

eng/_util/cmd/build/build.go

@@ -41,6 +41,7 @@ func main() {
 	o := &options{}
 
 	flag.BoolVar(&o.SkipBuild, "skipbuild", false, "Disable building Go.")
+	flag.BoolVar(&o.SkipBuildRace, "skipbuildrace", false, "Disable building Go with race detector.")
 	flag.BoolVar(&o.Test, "test", false, "Enable running tests.")
 	flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.")
 	flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.")
@@ -78,13 +79,14 @@ func main() {
 }
 
 type options struct {
-	SkipBuild  bool
-	Test       bool
-	PackBuild  bool
-	PackSource bool
-	CreatePDB  bool
-	Refresh    bool
-	Experiment string
+	SkipBuild     bool
+	SkipBuildRace bool
+	Test          bool
+	PackBuild     bool
+	PackSource    bool
+	CreatePDB     bool
+	Refresh       bool
+	Experiment    string
 
 	TestJSONFlags *buildutil.TestJSONFlags
 
@@ -202,7 +204,12 @@ func build(o *options) (err error) {
 		// The race runtime requires cgo.
 		// It isn't supported on arm or 386.
 		// It's supported on arm64, but the official linux-arm64 distribution doesn't include it.
-		if os.Getenv("CGO_ENABLED") != "0" && targetArch != "arm" && targetArch != "arm64" && targetArch != "386" {
+		if !o.SkipBuildRace &&
+			os.Getenv("CGO_ENABLED") != "0" &&
+			targetArch != "arm" &&
+			targetArch != "arm64" &&
+			targetArch != "386" {
+
 			fmt.Println("---- Building race runtime...")
 			err := runCommandLine(
 				filepath.Join("..", "bin", "go"+executableExtension),

eng/_util/cmd/submodule-refresh/submodule-refresh.go

@@ -5,8 +5,11 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -22,6 +25,7 @@ applies patches to the stage by default, or optionally as commits.
 var (
 	commits          = flag.Bool("commits", false, "Apply the patches as commits.")
 	skipPatch        = flag.Bool("skip-patch", false, "Skip applying patches.")
+	take             = flag.Int("take", -1, "Only apply the first N patches. -1 means apply all.")
 	origin           = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
 	shallow          = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
 	fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
@@ -75,8 +79,51 @@ func refresh(rootDir string) error {
 		mode = patch.ApplyModeCommits
 	}
 
+	if *take >= 0 {
+		// The patch API applies all patches in a directory. To apply only the
+		// first N, copy them into a temporary directory and point the config there.
+		tmpDirRelative := filepath.Join("eng", "artifacts", "submodule-refresh", "patch-subset")
+		tmpDir := filepath.Join(config.RootDir, tmpDirRelative)
+		if err := os.RemoveAll(tmpDir); err != nil {
+			return err
+		}
+		if err := os.MkdirAll(tmpDir, 0o777); err != nil {
+			return err
+		}
+		i := 0
+		if err := patch.WalkGoPatches(config, func(path string) error {
+			if i >= *take {
+				log.Printf("Not including patch %q\n", path)
+				return nil
+			}
+			i++
+			log.Printf("Taking patch %q\n", path)
+			return copyFile(path, filepath.Join(tmpDir, filepath.Base(path)))
+		}); err != nil {
+			return err
+		}
+		if *take > i {
+			return fmt.Errorf("-take %d exceeds number of patches (%d)", *take, i)
+		}
+		config.PatchesDir = tmpDirRelative
+	}
+
 	if err := patch.Apply(config, mode); err != nil {
 		return err
 	}
 	return nil
 }
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	_, copyErr := io.Copy(out, in)
+	return errors.Join(copyErr, out.Close())
+}

eng/_util/cmd/upgrader/main.go eng/_util/cmd/upgrader/upgrader.goeng/_util/cmd/upgrader/main.go renamed to eng/_util/cmd/upgrader/upgrader.go

@@ -145,20 +145,55 @@ func upgradeDependency(info *depsInfo) error {
 	return nil
 }
 
-var goBinary = sync.OnceValue(func() string {
-	//lint:ignore SA1019 we want to know the binary that built us
-	//nolint:staticcheck // deprecated okay
-	path := filepath.Join(runtime.GOROOT(), "bin", "go")
+var repoRoot = sync.OnceValues(func() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("failed to get working directory: %w", err)
+	}
+	for {
+		if _, err := os.Stat(filepath.Join(dir, ".git-go-patch")); err == nil {
+			log.Printf("Found repo root: %s\n", dir)
+			return dir, nil
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return "", fmt.Errorf("could not find repo root: no .git-go-patch file found in any ancestor directory")
+		}
+		dir = parent
+	}
+})
+
+var goBinary = sync.OnceValues(func() (string, error) {
+	ext := ""
 	if runtime.GOOS == "windows" {
-		path += ".exe"
+		ext = ".exe"
 	}
+	// Use the dev-built Go binary, if it exists.
+	if root, err := repoRoot(); err == nil {
+		path := filepath.Join(root, "go", "bin", "go"+ext)
+		if _, err := os.Stat(path); err == nil {
+			log.Printf("Using %s\n", path)
+			return path, nil
+		}
+	}
+	//lint:ignore SA1019 we want to know the binary that built us
+	//nolint:staticcheck // deprecated okay
+	path := filepath.Join(runtime.GOROOT(), "bin", "go"+ext)
 	log.Printf("Using %s\n", path)
-	return path
+	return path, nil
 })
 
 func runGoCmd(wd string, args ...string) error {
-	cmd := exec.Command(goBinary(), args...)
-	cmd.Dir = wd
+	root, err := repoRoot()
+	if err != nil {
+		return err
+	}
+	binary, err := goBinary()
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command(binary, args...)
+	cmd.Dir = filepath.Join(root, wd)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("%q failed: %v: %s", cmd, err, out)