build: run LLVM package backends in parallel

cl/caller_tracking_precompute_test.go

@@ -71,3 +71,37 @@ func Logs() { dep.Where() }
 	}
 	wg.Wait()
 }
+
+func TestCallerTrackingPrecomputeRejectsLatePackages(t *testing.T) {
+	dep, root := buildCallerFrameSSAProgram(t,
+		"example.com/dep", `package dep
+func Where() {}
+`,
+		"example.com/root", `package root
+import "example.com/dep"
+func Logs() { dep.Where() }
+`)
+	tests := []struct {
+		name   string
+		lookup func(*CallerTracking, *gossa.Package)
+	}{
+		{name: "base", lookup: func(c *CallerTracking, pkg *gossa.Package) {
+			runtimeCallerBaseSet(c, pkg)
+		}},
+		{name: "extended", lookup: func(c *CallerTracking, pkg *gossa.Package) {
+			runtimeCallerFuncSet(c, pkg)
+		}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tracking := NewCallerTracking()
+			tracking.Precompute([]*gossa.Package{dep})
+			defer func() {
+				if recover() == nil {
+					t.Fatal("late caller-tracking lookup did not panic")
+				}
+			}()
+			test.lookup(tracking, root)
+		})
+	}
+}

cl/instr.go

@@ -927,6 +927,9 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function
 	if set, ok := c.extended[pkg]; ok {
 		return set
 	}
+	if c.precomputed {
+		panic("caller-tracking function set was not precomputed")
+	}
 	base := runtimeCallerBaseSet(c, pkg)
 	out := make(map[*ssa.Function]bool, len(base))
 	for fn := range base {
@@ -983,14 +986,18 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function
 // Precompute before workers start; recover facts also synchronize lazy queries
 // for nested and synthetic functions that are not package members.
 type CallerTracking struct {
-	base     map[*ssa.Package]map[*ssa.Function]bool
-	extended map[*ssa.Package]map[*ssa.Function]bool
-	recover  *recoverFacts
+	base        map[*ssa.Package]map[*ssa.Function]bool
+	extended    map[*ssa.Package]map[*ssa.Function]bool
+	recover     *recoverFacts
+	precomputed bool
 }
 
 // Precompute resolves caller-tracking and recover data before package backends
 // start. Once it returns, callers may share c for concurrent lookups as long as
-// pkgs contains every package that can be passed to this compilation.
+// pkgs contains every package that can be passed to this compilation. A later
+// caller-tracking lookup miss panics instead of mutating the shared maps;
+// recover facts retain their own synchronization for nested and synthetic
+// functions that are not package members.
 func (c *CallerTracking) Precompute(pkgs []*ssa.Package) {
 	if c == nil {
 		return
@@ -1006,6 +1013,7 @@ func (c *CallerTracking) Precompute(pkgs []*ssa.Package) {
 			runtimeCallerFuncSet(c, pkg)
 		}
 	}
+	c.precomputed = true
 }
 
 // NewCallerTracking creates the frontend-analysis caches for one compilation.
@@ -1035,6 +1043,9 @@ func runtimeCallerBaseSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function
 	if set, ok := c.base[pkg]; ok {
 		return set
 	}
+	if c.precomputed {
+		panic("caller-tracking base set was not precomputed")
+	}
 	set := computeRuntimeCallerBaseSet(pkg)
 	c.base[pkg] = set
 	return set

internal/build/backend_program_test.go

@@ -17,9 +17,13 @@
 package build
 
 import (
+	"errors"
+	"go/importer"
+	"go/types"
 	"runtime"
 	"testing"
 
+	"github.com/goplus/llgo/internal/packages"
 	llssa "github.com/goplus/llgo/ssa"
 )
 
@@ -46,3 +50,111 @@ func TestNewBackendSessionCreatesIndependentLLVMState(t *testing.T) {
 		t.Fatal("backend sessions share an LLVM context")
 	}
 }
+
+func addBackendProgramPackage(ctx *context, prog llssa.Program, path string) *aPackage {
+	source := &packages.Package{ID: path, PkgPath: path}
+	pkg := &aPackage{Package: source, LPkg: prog.NewPackage("p", path)}
+	ctx.pkgs[source] = pkg
+	return pkg
+}
+
+func TestBackendProgramsStayOwnedByPackagesUntilExplicitDispose(t *testing.T) {
+	coordinator := llssa.NewProgram(nil)
+	defer coordinator.Dispose()
+	ctx := &context{
+		prog: coordinator,
+		pkgs: make(map[*packages.Package]Package),
+	}
+	coordinatorPkg := addBackendProgramPackage(ctx, coordinator, "example.com/coordinator")
+	pkgs := make([]*aPackage, 2)
+	paths := []string{"example.com/p0", "example.com/p1"}
+	for i, path := range paths {
+		prog := llssa.NewProgram(nil)
+		pkgs[i] = addBackendProgramPackage(ctx, prog, path)
+	}
+	for _, pkg := range pkgs {
+		if pkg.LPkg == nil || pkg.LPkg.Module().IsNil() {
+			t.Fatal("retained package module was cleared before whole-program use")
+		}
+	}
+
+	ctx.disposeBackendPrograms()
+	for _, pkg := range pkgs {
+		if pkg.LPkg != nil {
+			t.Fatal("isolated package still references LPkg after dispose")
+		}
+	}
+	if coordinatorPkg.LPkg == nil {
+		t.Fatal("backend disposal cleared coordinator package")
+	}
+	ctx.disposeBackendPrograms()
+}
+
+func TestBackendAbiTypesFollowLinkedPackageOrder(t *testing.T) {
+	runtimePkg, err := importer.For("source", nil).Import(llssa.PkgRuntime)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newBackend := func(path, name string, raw types.Type) (llssa.Program, *aPackage) {
+		prog := llssa.NewProgram(nil)
+		prog.SetRuntime(runtimePkg)
+		pkg := prog.NewPackage("p", path)
+		pkg.RegisterAbiTypes([]llssa.AbiTypeInfo{{Name: name, Raw: raw}})
+		return prog, &aPackage{LPkg: pkg}
+	}
+
+	coordinator := llssa.NewProgram(nil)
+	defer coordinator.Dispose()
+	firstProg, first := newBackend("example.com/first", "z.type", types.Typ[types.Int])
+	defer firstProg.Dispose()
+	secondProg, second := newBackend("example.com/second", "a.type", types.Typ[types.String])
+	defer secondProg.Dispose()
+	duplicate := &aPackage{LPkg: firstProg.NewPackage("duplicate", "example.com/duplicate")}
+	coordinatorPkg := &aPackage{LPkg: coordinator.NewPackage("coordinator", "example.com/coordinator")}
+
+	ctx := &context{prog: coordinator}
+	infos := ctx.backendAbiTypes([]Package{nil, {}, coordinatorPkg, first, duplicate, second})
+	if len(infos) != 2 || infos[0].Name != "z.type" || infos[1].Name != "a.type" {
+		t.Fatalf("backend ABI types = %#v, want linked order [z.type a.type]", infos)
+	}
+}
+
+func TestBackendProgramsReleaseOnErrorAndPanic(t *testing.T) {
+	coordinator := llssa.NewProgram(nil)
+	defer coordinator.Dispose()
+	ctx := &context{
+		prog: coordinator,
+		pkgs: make(map[*packages.Package]Package),
+	}
+	retain := func(path string) *aPackage {
+		prog := llssa.NewProgram(nil)
+		return addBackendProgramPackage(ctx, prog, path)
+	}
+
+	errorPkg := retain("example.com/error")
+	wantErr := errors.New("link failed")
+	runError := func() (err error) {
+		defer ctx.disposeBackendPrograms()
+		return wantErr
+	}
+	if err := runError(); !errors.Is(err, wantErr) {
+		t.Fatalf("error unwind = %v, want %v", err, wantErr)
+	}
+	if errorPkg.LPkg != nil {
+		t.Fatal("error unwind retained a backend Program")
+	}
+
+	panicPkg := retain("example.com/panic")
+	func() {
+		defer func() {
+			if got := recover(); got != "link panic" {
+				t.Fatalf("panic unwind recovered %v", got)
+			}
+		}()
+		defer ctx.disposeBackendPrograms()
+		panic("link panic")
+	}()
+	if panicPkg.LPkg != nil {
+		t.Fatal("panic unwind retained a backend Program")
+	}
+}