test: qualify LLVM-built stdlib dependency closures

src/cmd/internal/testdir/llvm_stdlib_test.go

@@ -21,20 +21,14 @@ import (
 
 const llvmStdlibPolicyEnv = "GOALLC_RUN_LLVM_STDLIB"
 
-// A package is only a required LLVM standard library test after it survives
-// multiple independent test processes. The processes share the isolated build
-// cache below, so this repeats runtime qualification without recompiling every
-// package from scratch.
-const llvmStdlibWhitelistRuns = 3
-
 type llvmStdlibTestSet struct {
 	Whitelist         map[string]string            `json:"whitelist"`
 	Blacklist         map[string]string            `json:"blacklist"`
 	PlatformBlacklist map[string]map[string]string `json:"platform_blacklist,omitempty"`
 }
 
 type llvmStdlibPolicy struct {
-	EntryPackage llvmStdlibTestSet `json:"entry_package"`
+	Packages llvmStdlibTestSet `json:"packages"`
 }
 
 type llvmStdlibClass uint8
@@ -192,7 +186,7 @@ func validateLLVMStdlibPolicy(t *testing.T, packages map[string]bool, set llvmSt
 
 func TestLLVMStdlibPolicy(t *testing.T) {
 	packages := llvmStdlibPackages(t)
-	validateLLVMStdlibPolicy(t, packages, readLLVMStdlibPolicy(t).EntryPackage)
+	validateLLVMStdlibPolicy(t, packages, readLLVMStdlibPolicy(t).Packages)
 }
 
 func TestClassifyLLVMStdlibPackage(t *testing.T) {
@@ -234,6 +228,32 @@ func TestEffectiveLLVMStdlibTestSet(t *testing.T) {
 	}
 }
 
+func llvmStdlibDependencyPackages(t *testing.T, packages map[string]bool, name string) []string {
+	t.Helper()
+	cmd := testenv.Command(t, llvmStdlibGoTool(t), "list", "-deps", "-f={{.ImportPath}}", name)
+	cmd.Env = append(os.Environ(), "GOENV=off", "GOFLAGS=", "GOROOT="+testenv.GOROOT(t))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("list dependencies for standard library package %q: %v\n%s", name, err, out)
+	}
+	seen := make(map[string]bool)
+	var dependencies []string
+	for _, dependency := range strings.Fields(string(out)) {
+		if !packages[dependency] {
+			t.Fatalf("dependency-closure package %q has non-standard dependency %q", name, dependency)
+		}
+		if !seen[dependency] {
+			seen[dependency] = true
+			dependencies = append(dependencies, dependency)
+		}
+	}
+	if !seen[name] {
+		t.Fatalf("dependency closure for %q does not contain the package itself", name)
+	}
+	sort.Strings(dependencies)
+	return dependencies
+}
+
 func TestLLVMStdlib(t *testing.T) {
 	if os.Getenv(llvmStdlibPolicyEnv) != "1" {
 		t.Skipf("set %s=1 to run the LLVM standard library package policy", llvmStdlibPolicyEnv)
@@ -247,18 +267,25 @@ func TestLLVMStdlib(t *testing.T) {
 	}
 
 	packages := llvmStdlibPackages(t)
-	policySet := readLLVMStdlibPolicy(t).EntryPackage
+	policySet := readLLVMStdlibPolicy(t).Packages
 	validateLLVMStdlibPolicy(t, packages, policySet)
 	set := effectiveLLVMStdlibTestSet(policySet, platform)
 	configureLLVMTestToolchain(t)
 	toolexec := llvmToolexec(t, "default<O2>")
+	runtimeToolexec := llvmToolexecWithNativePackages(t, "default<O2>", "runtime_test", "runtime.test")
 
 	whitelist := make([]string, 0, len(set.Whitelist))
 	for name := range set.Whitelist {
 		whitelist = append(whitelist, name)
 	}
 	sort.Strings(whitelist)
-	t.Logf("LLVM standard library entry-package policy: %d white, %d black (%d packages)", len(whitelist), len(packages)-len(whitelist), len(packages))
+	t.Logf("LLVM standard library dependency-closure policy: %d white, %d black (%d packages)", len(whitelist), len(packages)-len(whitelist), len(packages))
+
+	dependencyPackages := make(map[string][]string, len(whitelist))
+	for _, name := range whitelist {
+		dependencyPackages[name] = llvmStdlibDependencyPackages(t, packages, name)
+		t.Logf("LLVM stdlib dependency closure: package=%q packages=%d", name, len(dependencyPackages[name]))
+	}
 
 	knownBlacklist := make([]string, 0, len(set.Blacklist)-1)
 	for name := range set.Blacklist {
@@ -279,33 +306,52 @@ func TestLLVMStdlib(t *testing.T) {
 	cache := t.TempDir()
 	for _, name := range whitelist {
 		t.Run(name, func(t *testing.T) {
-			for run := 1; run <= llvmStdlibWhitelistRuns; run++ {
-				ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 5*time.Minute)
-				cmd := testenv.CommandContext(t, ctx, llvmStdlibGoTool(t),
-					"test",
-					"-count=1",
-					"-timeout=2m",
-					"-toolexec="+toolexec,
-					fmt.Sprintf("-gcflags=%s=-enablellvm -llvmironly", name),
-					name,
-				)
-				cmd.Env = append(os.Environ(),
-					"GOENV=off",
-					"GOFLAGS=",
-					"GOROOT="+testenv.GOROOT(t),
-					"GOCACHE="+cache,
-				)
-				out, err := cmd.CombinedOutput()
-				ctxErr := ctx.Err()
-				cancel()
-				if err != nil {
-					if ctxErr != nil {
-						t.Fatalf("LLVM stdlib whitelist result: TIMEOUT package=%q run=%d/%d: %v\n%s", name, run, llvmStdlibWhitelistRuns, ctxErr, out)
-					}
-					t.Fatalf("LLVM stdlib whitelist result: FAIL package=%q run=%d/%d: %v\n%s", name, run, llvmStdlibWhitelistRuns, err, out)
+			compilePackages := dependencyPackages[name]
+			packageToolexec := toolexec
+			testTimeout := "2m"
+			processTimeout := 5 * time.Minute
+			if name == "runtime" {
+				testTimeout = "5m"
+				processTimeout = 8 * time.Minute
+				// runtime_test and the generated runtime.test main are test
+				// scaffolding rather than part of the qualified runtime closure.
+				packageToolexec = runtimeToolexec
+			}
+			ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), processTimeout)
+			args := []string{
+				"test",
+				"-count=1",
+				"-timeout=" + testTimeout,
+				"-toolexec=" + packageToolexec,
+			}
+			if name == "runtime" {
+				// LLVM GoObj does not yet emit the complete per-function
+				// DWARF carrier set expected by the Go linker. Runtime
+				// qualification currently covers code generation, GoObj,
+				// linking, and execution, but not debug information.
+				args = append(args, "-ldflags=-w")
+			}
+			for _, compilePackage := range compilePackages {
+				args = append(args, fmt.Sprintf("-gcflags=%s=-enablellvm -llvmironly", compilePackage))
+			}
+			args = append(args, name)
+			cmd := testenv.CommandContext(t, ctx, llvmStdlibGoTool(t), args...)
+			cmd.Env = append(os.Environ(),
+				"GOENV=off",
+				"GOFLAGS=",
+				"GOROOT="+testenv.GOROOT(t),
+				"GOCACHE="+cache,
+			)
+			out, err := cmd.CombinedOutput()
+			ctxErr := ctx.Err()
+			cancel()
+			if err != nil {
+				if ctxErr != nil {
+					t.Fatalf("LLVM stdlib whitelist result: TIMEOUT package=%q: %v\n%s", name, ctxErr, out)
 				}
-				t.Logf("LLVM stdlib whitelist result: PASS package=%q run=%d/%d", name, run, llvmStdlibWhitelistRuns)
+				t.Fatalf("LLVM stdlib whitelist result: FAIL package=%q: %v\n%s", name, err, out)
 			}
+			t.Logf("LLVM stdlib whitelist result: PASS package=%q", name)
 		})
 	}
 }

src/cmd/internal/testdir/llvm_test.go

@@ -886,6 +886,10 @@ func runLLVMWriteBarrierIRTests(t *testing.T) {
 }
 
 func llvmToolexec(t *testing.T, optPasses string) string {
+	return llvmToolexecWithNativePackages(t, optPasses)
+}
+
+func llvmToolexecWithNativePackages(t *testing.T, optPasses string, nativePackages ...string) string {
 	t.Helper()
 	wrapper := llvmToolexecPath(t)
 
@@ -900,6 +904,9 @@ func llvmToolexec(t *testing.T, optPasses string) string {
 		opt := llvmToolPath(t, "opt", "GOALLC_OPT")
 		args = append(args, "-opt="+opt, "-opt-passes="+optPasses)
 	}
+	for _, name := range nativePackages {
+		args = append(args, "-native-package="+name)
+	}
 	value, err := quoted.Join(args)
 	if err != nil {
 		t.Fatal(err)

src/cmd/llvmtoolexec/main.go

@@ -21,18 +21,49 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 )
 
+type stringSetFlag map[string]struct{}
+
+func (f *stringSetFlag) Set(value string) error {
+	if value == "" {
+		return errors.New("package path must not be empty")
+	}
+	if *f == nil {
+		*f = make(map[string]struct{})
+	}
+	(*f)[value] = struct{}{}
+	return nil
+}
+
+func (f *stringSetFlag) String() string {
+	if f == nil {
+		return ""
+	}
+	values := make([]string, 0, len(*f))
+	for value := range *f {
+		values = append(values, value)
+	}
+	sort.Strings(values)
+	return strings.Join(values, ",")
+}
+
 var (
 	llcPath        = flag.String("llc", os.Getenv("GOALLC_LLC"), "path to llc")
 	optPath        = flag.String("opt", os.Getenv("GOALLC_OPT"), "path to opt")
 	optPasses      = flag.String("opt-passes", "", "optional LLVM optimization pipeline to run before llc")
 	passPluginPath = flag.String("pass-plugin", os.Getenv("GOALLC_PASS_PLUGIN"), "path to the GoALLC LLVM pass plugin (default next to llc)")
 	keepIR         = flag.Bool("keep-ir", false, "keep the compiler-generated .ll sidecar")
+	nativePackages stringSetFlag
 )
 
+func init() {
+	flag.Var(&nativePackages, "native-package", "compile this exact -p package with the native Go backend even when inherited gcflags select LLVM (repeatable)")
+}
+
 func main() {
 	flag.Parse()
 	if flag.NArg() < 1 {
@@ -55,6 +86,10 @@ func main() {
 		run(tool, args...)
 		return
 	}
+	if useNativeCompiler(args, nativePackages) {
+		run(tool, withoutLLVMCompileFlags(args)...)
+		return
+	}
 	if !isCompileAction(args) {
 		run(tool, args...)
 		return
@@ -290,6 +325,8 @@ func printToolIdentity(tool string, args []string, llc, configuredOpt, optPasses
 	identityInput := append([]byte(nil), out...)
 	identityInput = append(identityInput, "\x00opt-passes="...)
 	identityInput = append(identityInput, optPasses...)
+	identityInput = append(identityInput, "\x00native-packages="...)
+	identityInput = append(identityInput, nativePackages.String()...)
 	identity, err := backendIdentity(identityInput, append([]string{wrapper}, backendFiles...)...)
 	if err != nil {
 		fatalf("computing backend identity: %v", err)
@@ -426,6 +463,27 @@ func toolFlag(args []string, name string) (string, bool) {
 	return "", false
 }
 
+func useNativeCompiler(args []string, packages stringSetFlag) bool {
+	pkg, ok := toolFlag(args, "-p")
+	if !ok {
+		return false
+	}
+	_, ok = packages[pkg]
+	return ok
+}
+
+func withoutLLVMCompileFlags(args []string) []string {
+	native := make([]string, 0, len(args))
+	for _, arg := range args {
+		if arg == "-enablellvm" || strings.HasPrefix(arg, "-enablellvm=") ||
+			arg == "-llvmironly" || strings.HasPrefix(arg, "-llvmironly=") {
+			continue
+		}
+		native = append(native, arg)
+	}
+	return native
+}
+
 func run(path string, args ...string) {
 	cmd := exec.Command(path, args...)
 	cmd.Stdin = os.Stdin

src/cmd/llvmtoolexec/main_test.go

@@ -384,6 +384,27 @@ func TestCompileInvocationClassification(t *testing.T) {
 	}
 }
 
+func TestNativePackageOverride(t *testing.T) {
+	packages := stringSetFlag{"runtime_test": {}}
+	args := []string{
+		"-p", "runtime_test", "-enablellvm", "-llvmironly=true",
+		"-o", "out.a", "callers_test.go",
+	}
+	if !useNativeCompiler(args, packages) {
+		t.Fatal("exact native package was not recognized")
+	}
+	native := withoutLLVMCompileFlags(args)
+	if hasLLVMCompileFlags(native) {
+		t.Fatalf("LLVM selection survived native override: %q", native)
+	}
+	if got, ok := toolFlag(native, "-p"); !ok || got != "runtime_test" {
+		t.Fatalf("native override changed package flag to %q, %v", got, ok)
+	}
+	if useNativeCompiler([]string{"-p=runtime", "-enablellvm", "-llvmironly"}, packages) {
+		t.Fatal("native package override matched a different package")
+	}
+}
+
 func TestBoolToolFlag(t *testing.T) {
 	tests := []struct {
 		name string

src/runtime/export_test.go

@@ -607,9 +607,23 @@ func G0StackOverflow() {
 	})
 }
 
+var stackOverflowTestState uint32
+
 func stackOverflow(x *byte) {
 	var buf [256]byte
+	// Keep this recursion from becoming a tail-recursive loop. The test needs
+	// real stack growth, but the LLVM backend is otherwise free to optimize tail
+	// calls more aggressively than the native Go backend. An atomic load makes
+	// the return path reachable to the optimizer, and the post-call atomic
+	// operation keeps state from the current frame live across the call.
+	buf[0] = byte(atomic.Load(&stackOverflowTestState))
+	if buf[0] != 0 {
+		return
+	}
 	stackOverflow(&buf[0])
+	if x != nil {
+		atomic.Xadd(&stackOverflowTestState, int32(*x))
+	}
 }
 
 func RunGetgThreadSwitchTest() {