diff --git a/cl/caller_tracking_precompute_test.go b/cl/caller_tracking_precompute_test.go index 419c46c88d..9413110a82 100644 --- a/cl/caller_tracking_precompute_test.go +++ b/cl/caller_tracking_precompute_test.go @@ -71,3 +71,37 @@ func Logs() { dep.Where() } } wg.Wait() } + +func TestCallerTrackingPrecomputeRejectsLatePackages(t *testing.T) { + dep, root := buildCallerFrameSSAProgram(t, + "example.com/dep", `package dep +func Where() {} +`, + "example.com/root", `package root +import "example.com/dep" +func Logs() { dep.Where() } +`) + tests := []struct { + name string + lookup func(*CallerTracking, *gossa.Package) + }{ + {name: "base", lookup: func(c *CallerTracking, pkg *gossa.Package) { + runtimeCallerBaseSet(c, pkg) + }}, + {name: "extended", lookup: func(c *CallerTracking, pkg *gossa.Package) { + runtimeCallerFuncSet(c, pkg) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tracking := NewCallerTracking() + tracking.Precompute([]*gossa.Package{dep}) + defer func() { + if recover() == nil { + t.Fatal("late caller-tracking lookup did not panic") + } + }() + test.lookup(tracking, root) + }) + } +} diff --git a/cl/instr.go b/cl/instr.go index c649fe0ee2..aa6a95a9d6 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -927,6 +927,9 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.extended[pkg]; ok { return set } + if c.precomputed { + panic("caller-tracking function set was not precomputed") + } base := runtimeCallerBaseSet(c, pkg) out := make(map[*ssa.Function]bool, len(base)) for fn := range base { @@ -983,14 +986,18 @@ func runtimeCallerFuncSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function // Precompute before workers start; recover facts also synchronize lazy queries // for nested and synthetic functions that are not package members. type CallerTracking struct { - base map[*ssa.Package]map[*ssa.Function]bool - extended map[*ssa.Package]map[*ssa.Function]bool - recover *recoverFacts + base map[*ssa.Package]map[*ssa.Function]bool + extended map[*ssa.Package]map[*ssa.Function]bool + recover *recoverFacts + precomputed bool } // Precompute resolves caller-tracking and recover data before package backends // start. Once it returns, callers may share c for concurrent lookups as long as -// pkgs contains every package that can be passed to this compilation. +// pkgs contains every package that can be passed to this compilation. A later +// caller-tracking lookup miss panics instead of mutating the shared maps; +// recover facts retain their own synchronization for nested and synthetic +// functions that are not package members. func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { if c == nil { return @@ -1006,6 +1013,7 @@ func (c *CallerTracking) Precompute(pkgs []*ssa.Package) { runtimeCallerFuncSet(c, pkg) } } + c.precomputed = true } // NewCallerTracking creates the frontend-analysis caches for one compilation. @@ -1035,6 +1043,9 @@ func runtimeCallerBaseSet(c *CallerTracking, pkg *ssa.Package) map[*ssa.Function if set, ok := c.base[pkg]; ok { return set } + if c.precomputed { + panic("caller-tracking base set was not precomputed") + } set := computeRuntimeCallerBaseSet(pkg) c.base[pkg] = set return set diff --git a/internal/build/backend_program_test.go b/internal/build/backend_program_test.go index 7af22f6416..fc13f4a4e4 100644 --- a/internal/build/backend_program_test.go +++ b/internal/build/backend_program_test.go @@ -17,9 +17,13 @@ package build import ( + "errors" + "go/importer" + "go/types" "runtime" "testing" + "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" ) @@ -46,3 +50,111 @@ func TestNewBackendSessionCreatesIndependentLLVMState(t *testing.T) { t.Fatal("backend sessions share an LLVM context") } } + +func addBackendProgramPackage(ctx *context, prog llssa.Program, path string) *aPackage { + source := &packages.Package{ID: path, PkgPath: path} + pkg := &aPackage{Package: source, LPkg: prog.NewPackage("p", path)} + ctx.pkgs[source] = pkg + return pkg +} + +func TestBackendProgramsStayOwnedByPackagesUntilExplicitDispose(t *testing.T) { + coordinator := llssa.NewProgram(nil) + defer coordinator.Dispose() + ctx := &context{ + prog: coordinator, + pkgs: make(map[*packages.Package]Package), + } + coordinatorPkg := addBackendProgramPackage(ctx, coordinator, "example.com/coordinator") + pkgs := make([]*aPackage, 2) + paths := []string{"example.com/p0", "example.com/p1"} + for i, path := range paths { + prog := llssa.NewProgram(nil) + pkgs[i] = addBackendProgramPackage(ctx, prog, path) + } + for _, pkg := range pkgs { + if pkg.LPkg == nil || pkg.LPkg.Module().IsNil() { + t.Fatal("retained package module was cleared before whole-program use") + } + } + + ctx.disposeBackendPrograms() + for _, pkg := range pkgs { + if pkg.LPkg != nil { + t.Fatal("isolated package still references LPkg after dispose") + } + } + if coordinatorPkg.LPkg == nil { + t.Fatal("backend disposal cleared coordinator package") + } + ctx.disposeBackendPrograms() +} + +func TestBackendAbiTypesFollowLinkedPackageOrder(t *testing.T) { + runtimePkg, err := importer.For("source", nil).Import(llssa.PkgRuntime) + if err != nil { + t.Fatal(err) + } + newBackend := func(path, name string, raw types.Type) (llssa.Program, *aPackage) { + prog := llssa.NewProgram(nil) + prog.SetRuntime(runtimePkg) + pkg := prog.NewPackage("p", path) + pkg.RegisterAbiTypes([]llssa.AbiTypeInfo{{Name: name, Raw: raw}}) + return prog, &aPackage{LPkg: pkg} + } + + coordinator := llssa.NewProgram(nil) + defer coordinator.Dispose() + firstProg, first := newBackend("example.com/first", "z.type", types.Typ[types.Int]) + defer firstProg.Dispose() + secondProg, second := newBackend("example.com/second", "a.type", types.Typ[types.String]) + defer secondProg.Dispose() + duplicate := &aPackage{LPkg: firstProg.NewPackage("duplicate", "example.com/duplicate")} + coordinatorPkg := &aPackage{LPkg: coordinator.NewPackage("coordinator", "example.com/coordinator")} + + ctx := &context{prog: coordinator} + infos := ctx.backendAbiTypes([]Package{nil, {}, coordinatorPkg, first, duplicate, second}) + if len(infos) != 2 || infos[0].Name != "z.type" || infos[1].Name != "a.type" { + t.Fatalf("backend ABI types = %#v, want linked order [z.type a.type]", infos) + } +} + +func TestBackendProgramsReleaseOnErrorAndPanic(t *testing.T) { + coordinator := llssa.NewProgram(nil) + defer coordinator.Dispose() + ctx := &context{ + prog: coordinator, + pkgs: make(map[*packages.Package]Package), + } + retain := func(path string) *aPackage { + prog := llssa.NewProgram(nil) + return addBackendProgramPackage(ctx, prog, path) + } + + errorPkg := retain("example.com/error") + wantErr := errors.New("link failed") + runError := func() (err error) { + defer ctx.disposeBackendPrograms() + return wantErr + } + if err := runError(); !errors.Is(err, wantErr) { + t.Fatalf("error unwind = %v, want %v", err, wantErr) + } + if errorPkg.LPkg != nil { + t.Fatal("error unwind retained a backend Program") + } + + panicPkg := retain("example.com/panic") + func() { + defer func() { + if got := recover(); got != "link panic" { + t.Fatalf("panic unwind recovered %v", got) + } + }() + defer ctx.disposeBackendPrograms() + panic("link panic") + }() + if panicPkg.LPkg != nil { + t.Fatal("panic unwind retained a backend Program") + } +} diff --git a/internal/build/build.go b/internal/build/build.go index 6db6ae9085..0b0ccaacee 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -612,10 +612,11 @@ func Build(inv Invocation) ([]Package, error) { output := conf.OutFile != "" ctx := &context{conf: cfg, progSSA: progSSA, prog: prog, dedup: dedup, patches: patches, callerTracking: cl.NewCallerTracking(), - built: make(map[string]none), initial: initial, mode: mode, + initial: initial, mode: mode, fingerprinting: make(map[string]bool), pkgs: map[*packages.Package]Package{}, pkgByID: map[string]Package{}, + cacheManager: newCacheManager(), output: output, passOpt: passOpt, buildConf: conf, @@ -626,6 +627,10 @@ func Build(inv Invocation) ([]Package, error) { } defer ctx.closePackageMetas() defer ctx.closePackageArchiveBuffers() + // Isolated backends use independent LLVM contexts. Keep Programs needed by + // whole-program consumers alive through deadcode analysis and strong ABI type + // override emission, then release them on every normal, error, or panic path. + defer ctx.disposeBackendPrograms() // default runtime globals must be registered before packages are built addGlobalString(conf, "runtime.defaultGOROOT="+runtime.GOROOT(), nil) @@ -743,6 +748,7 @@ func Build(inv Invocation) ([]Package, error) { } } } + ctx.disposeBackendPrograms() if mode == ModeTest && ctx.testFail { mockable.Exit(1) @@ -870,7 +876,6 @@ type context struct { dedup packages.Deduper patches cl.Patches callerTracking *cl.CallerTracking - built map[string]none fingerprinting map[string]bool cacheDisabled map[string]none initial []*packages.Package @@ -895,18 +900,64 @@ type context struct { llvmVersion string // go list derived file lists (SFiles, etc.) - sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths + sfilesCache map[string][]string // pkg.ID -> absolute .s/.S file paths + sfilesFrozen bool // plan9asm package policy parsed from env. - plan9asmOnce sync.Once - plan9asmMode plan9asmPkgsEnvMode - plan9asmPkgs map[string]bool + plan9asmOnce sync.Once + plan9asmReady bool + plan9asmMode plan9asmPkgsEnvMode + plan9asmPkgs map[string]bool // pclnExternal is populated while generating the synthetic main module // and completed with final linked PCs by the post-link externalizer. pclnExternal *pclnmap.Data } +// backendAbiTypes returns Go-owned type identities from isolated Programs in +// stable linked-package order. The Programs remain alive while the entry +// module recreates target-local declarations, but no LLVM value crosses a +// Context boundary. +func (c *context) backendAbiTypes(pkgs []Package) []llssa.AbiTypeInfo { + seen := make(map[llssa.Program]none) + var infos []llssa.AbiTypeInfo + for _, pkg := range pkgs { + if pkg == nil || pkg.LPkg == nil { + continue + } + prog := pkg.LPkg.Prog + if prog == nil || prog == c.prog { + continue + } + if _, ok := seen[prog]; ok { + continue + } + seen[prog] = none{} + infos = append(infos, prog.AbiTypes()...) + } + return infos +} + +func (c *context) disposeBackendPrograms() { + programs := make(map[llssa.Program]none) + // Clear every package reference before destroying any LLVM context so no + // later observer can retain a dangling cross-context module. + for _, pkg := range c.pkgs { + if pkg == nil || pkg.LPkg == nil { + continue + } + prog := pkg.LPkg.Prog + if prog == nil || prog == c.prog { + continue + } + programs[prog] = none{} + pkg.LPkg = nil + } + for prog := range programs { + prog.Dispose() + } +} + // closePackageMetas releases metadata mappings owned by this build. Metadata // remains available to hooks and whole-program consumers until Do returns. func (c *context) closePackageMetas() { @@ -1033,65 +1084,42 @@ func normalizeToArchive(ctx *context, aPkg *aPackage, verbose bool) error { } func buildAllPkgs(ctx *context, pkgs []*aPackage, verbose bool) ([]*aPackage, error) { - // Split packages into runtime tree vs others so we can defer runtime build. - var runtimePkgs []*packageBuildTask - var normalPkgs []*packageBuildTask + // Split packages into runtime tree vs others so runtime preparation remains + // deferred until ordinary package results show that it is needed. + var runtimeTasks []*packageBuildTask + var normalTasks []*packageBuildTask for _, p := range pkgs { task := newPackageBuildTask(p) if task.isRuntime() { - runtimePkgs = append(runtimePkgs, task) + runtimeTasks = append(runtimeTasks, task) } else { - normalPkgs = append(normalPkgs, task) + normalTasks = append(normalTasks, task) } } - - var needRuntime, needPyInit bool + // Resolve the lazy Plan 9 policy before workers start. + _ = ctx.plan9asmEnabled("") // Build non-runtime packages first, so we know whether runtime is actually needed. - for _, task := range normalPkgs { - result, err := buildOnePackage(ctx, task, verbose) - if err != nil { - return nil, err - } - needRuntime = needRuntime || result.needRuntime - needPyInit = needPyInit || result.needPyInit + if err := buildPackageGroup(ctx, normalTasks, verbose); err != nil { + return nil, err } + needRuntime, needPyInit := packageRuntimeNeeds(normalTasks) // Only build runtime packages when required (or host build with empty Target). if needRuntime || needPyInit || ctx.buildConf.Target == "" { - for _, task := range runtimePkgs { - if _, err := buildOnePackage(ctx, task, verbose); err != nil { - return nil, err - } + if err := buildPackageGroup(ctx, runtimeTasks, verbose); err != nil { + return nil, err } } return pkgs, nil } -// buildOnePackage is the serial package pipeline. Its explicit stages are the -// contract used by later package workers; this commit deliberately preserves -// serial LLVM execution. -func buildOnePackage(ctx *context, task *packageBuildTask, verbose bool) (packageBuildResult, error) { - if err := prePackageBuild(ctx, task, verbose); err != nil || task.skip { - return packageBuildResultFor(task), err - } - if err := executePackageBuild(ctx, task, verbose); err != nil { - return packageBuildResultFor(task), err - } - return finalizePackageBuild(ctx, task, verbose) -} - // prePackageBuild performs classification, fingerprinting, and cache // lookup without creating or transforming an LLVM module. func prePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { aPkg := task.pkg pkg := aPkg.Package - if _, ok := ctx.built[pkg.ID]; ok { - task.skip = true - return nil - } - ctx.built[pkg.ID] = none{} if task.isDeclOnly() { pkg.ExportFile = "" task.skip = true @@ -1125,7 +1153,7 @@ func executePackageBuild(ctx *context, task *packageBuildTask, verbose bool) err if err := buildPkg(ctx, aPkg, verbose); err != nil { return err } - if task.needsRuntimeSignals() { + if task.needsRuntimeSignals() && aPkg.LPkg != nil { aPkg.setNeedRuntimeOrPyInit(aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit) } return nil @@ -1133,13 +1161,13 @@ func executePackageBuild(ctx *context, task *packageBuildTask, verbose bool) err // finalizePackageBuild publishes the archive and cache metadata. Cache hits // already carry both and therefore require no publication. -func finalizePackageBuild(ctx *context, task *packageBuildTask, verbose bool) (packageBuildResult, error) { +func finalizePackageBuild(ctx *context, task *packageBuildTask, verbose bool) error { aPkg := task.pkg if aPkg.CacheHit { - return packageBuildResultFor(task), nil + return nil } if err := normalizeToArchive(ctx, aPkg, verbose); err != nil { - return packageBuildResultFor(task), err + return err } if task.kind == cl.PkgLinkExtern { appendExternalLinkArgs(ctx, aPkg, task.kindParam) @@ -1147,7 +1175,7 @@ func finalizePackageBuild(ctx *context, task *packageBuildTask, verbose bool) (p if err := ctx.saveToCache(aPkg); err != nil && verbose { fmt.Fprintf(os.Stderr, "warning: failed to save cache for %s: %v\n", aPkg.PkgPath, err) } - return packageBuildResultFor(task), nil + return nil } func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { @@ -1159,7 +1187,7 @@ func appendExternalLinkArgs(ctx *context, aPkg *aPackage, spec string) { for _, alt := range altParts { alt = strings.TrimSpace(alt) if strings.ContainsRune(alt, '$') { - expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(alt)...) + expdArgs = append(expdArgs, xenv.ExpandEnvToArgsWith(alt, ctx.commands.dir, ctx.commands.environ)...) atomic.AddInt32(&ctx.nLibdir, 1) } else { fields := strings.Fields(alt) @@ -1457,6 +1485,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa methodByIndex: methodByIndex, methodByName: methodByName, abiSymbols: linkedModuleGlobals(linkedOrder), + abiTypes: ctx.backendAbiTypes(linkedOrder), funcInfo: funcInfo, pcLineInfo: pcLineInfo, }) @@ -2721,7 +2750,7 @@ func clFiles(ctx *context, files string, pkg *packages.Package, procFile func(li args := make([]string, 0, 16) if strings.HasPrefix(files, "$") { // has cflags if pos := strings.IndexByte(files, ':'); pos > 0 { - cflags := xenv.ExpandEnvToArgs(files[:pos]) + cflags := xenv.ExpandEnvToArgsWith(files[:pos], ctx.commands.dir, ctx.commands.environ) files = files[pos+1:] args = append(args, cflags...) } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 74951d5cd1..1df520a55f 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -216,6 +216,111 @@ func TestConcurrentInvocationsIsolateFrontendOptions(t *testing.T) { } } +func TestConcurrentDeadcodeBuildsUseIndependentWorkerContexts(t *testing.T) { + if !buildenv.Dev { + t.Skip("deadcode drop requires a development build") + } + repoRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + fixture := filepath.Join(repoRoot, "cl", "_testdrop", "direct_method") + t.Setenv("LLGO_ROOT", repoRoot) + t.Setenv(llgoBuildCache, "0") + + type result struct { + output string + err error + } + results := make(chan result, 2) + for i := range 2 { + output := filepath.Join(t.TempDir(), fmt.Sprintf("direct-method-%d", i)) + conf := NewDefaultConf(ModeBuild) + conf.DeadcodeDrop = true + conf.PCLNMode = PCLNNone + conf.BuildParallelism = 2 + conf.OutFile = output + go func() { + _, err := Build(Invocation{Args: []string{"."}, Config: conf, Dir: fixture}) + results <- result{output: output, err: err} + }() + } + + for range 2 { + got := <-results + if got.err != nil { + t.Fatal(got.err) + } + data, err := exec.Command(got.output).CombinedOutput() + if err != nil { + t.Fatalf("run %s: %v\n%s", got.output, err, data) + } + if strings.TrimSpace(string(data)) != "42" { + t.Fatalf("run %s output = %q, want 42", got.output, data) + } + } +} + +func TestDeadcodeBuildColdAndHotPackageCache(t *testing.T) { + if !buildenv.Dev { + t.Skip("deadcode drop requires a development build") + } + repoRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + fixture := filepath.Join(repoRoot, "cl", "_testdrop", "direct_method") + t.Setenv("LLGO_ROOT", repoRoot) + t.Setenv(llgoBuildCache, "1") + + for _, parallelism := range []int{1, 8} { + t.Run(fmt.Sprintf("p%d", parallelism), func(t *testing.T) { + oldCacheRoot := cacheRootFunc + cacheDir := t.TempDir() + cacheRootFunc = func() string { return cacheDir } + defer func() { cacheRootFunc = oldCacheRoot }() + + build := func(name string) []Package { + output := filepath.Join(t.TempDir(), name) + conf := NewDefaultConf(ModeBuild) + conf.DeadcodeDrop = true + conf.PCLNMode = PCLNNone + conf.BuildParallelism = parallelism + conf.OutFile = output + pkgs, err := Build(Invocation{Args: []string{"."}, Config: conf, Dir: fixture}) + if err != nil { + t.Fatal(err) + } + data, err := exec.Command(output).CombinedOutput() + if err != nil { + t.Fatalf("run %s: %v\n%s", output, err, data) + } + if strings.TrimSpace(string(data)) != "42" { + t.Fatalf("run %s output = %q, want 42", output, data) + } + return pkgs + } + + first := build("cold") + for _, pkg := range first { + if pkg.CacheHit { + t.Fatalf("cold build unexpectedly hit package cache for %s", pkg.PkgPath) + } + } + second := build("hot") + cacheHits := 0 + for _, pkg := range second { + if pkg.CacheHit { + cacheHits++ + } + } + if cacheHits == 0 { + t.Fatal("hot build did not reuse any package archives") + } + }) + } +} + func TestResolveOutputsUsesInvocationDirectory(t *testing.T) { dir := t.TempDir() out := &OutFmtDetails{ diff --git a/internal/build/deadcode_test.go b/internal/build/deadcode_test.go index 41951fe373..f54f5ec870 100644 --- a/internal/build/deadcode_test.go +++ b/internal/build/deadcode_test.go @@ -21,8 +21,13 @@ func TestApplyDeadcodeDropOverridesWritesStrongTypeOverride(t *testing.T) { Goarch: "amd64", }, } + defer ctx.prog.Dispose() - srcPkg := ctx.prog.NewPackage("pkg", "pkg") + // Isolated package workers and the synthetic entry module deliberately use + // different LLVM contexts. Keep this test faithful to that production path. + srcProg := llssa.NewProgram(nil) + defer srcProg.Dispose() + srcPkg := srcProg.NewPackage("pkg", "pkg") addMethodTypeGlobal(srcPkg.Module(), "_llgo_pkg.T") pkgMeta := buildDeadcodeMeta(t) defer pkgMeta.Close() @@ -50,6 +55,9 @@ func TestApplyDeadcodeDropOverridesWritesStrongTypeOverride(t *testing.T) { if strings.Contains(out, `ptr @"pkg.(*T).N"`) || strings.Contains(out, `ptr @pkg.T.N`) { t.Fatalf("dead method slot still references N functions:\n%s", out) } + if err := llvm.VerifyModule(entryPkg.LPkg.Module(), llvm.ReturnStatusAction); err != nil { + t.Fatalf("cross-context strong type override produced invalid entry module: %v\n%s", err, out) + } } func TestDCEEntryRootCandidates(t *testing.T) { diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 2693cb8ed4..30c5046eb6 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -43,6 +43,7 @@ type genConfig struct { methodByIndex map[int]none methodByName map[string]none abiSymbols map[string]none + abiTypes []llssa.AbiTypeInfo funcInfo []funcInfoRecord pcLineInfo []pcLineRecord } @@ -94,6 +95,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g var abiInit llssa.Function if cfg.abiInit != 0 { + mainPkg.RegisterAbiTypes(cfg.abiTypes) abiInit = mainPkg.InitAbiTypesFor("init$abitypes", func(sym *llssa.AbiSymbol) bool { if _, ok := cfg.abiSymbols[sym.Name]; !ok { return false diff --git a/internal/build/package_archive_test.go b/internal/build/package_archive_test.go index f76987b88b..d20e11f2d4 100644 --- a/internal/build/package_archive_test.go +++ b/internal/build/package_archive_test.go @@ -157,7 +157,7 @@ func TestNormalizeToArchiveFailsWithoutObjectFallback(t *testing.T) { buffer: memoryBuf, }}, } - _, err := finalizePackageBuild(ctx, &packageBuildTask{pkg: pkg}, false) + err := finalizePackageBuild(ctx, &packageBuildTask{pkg: pkg}, false) if err == nil { t.Fatal("finalizePackageBuild succeeded with a missing member") } diff --git a/internal/build/package_build.go b/internal/build/package_build.go index 12942f2c68..6281c94433 100644 --- a/internal/build/package_build.go +++ b/internal/build/package_build.go @@ -16,13 +16,21 @@ package build -import "github.com/goplus/llgo/cl" +import ( + "fmt" + "sync" + + "github.com/goplus/llgo/cl" + "github.com/goplus/llgo/internal/packages" +) type packageBuildTask struct { pkg *aPackage kind int kindParam string skip bool + isolated bool + parallel bool } func newPackageBuildTask(pkg *aPackage) *packageBuildTask { @@ -54,14 +62,223 @@ func (t *packageBuildTask) needsRuntimeSignals() bool { return !t.isLinkOnly() && !t.isDeclOnly() } -type packageBuildResult struct { - needRuntime bool - needPyInit bool +// preparePackageBuilds completes every coordinator-only read before workers +// start. In particular, packageRequiresCoordinator populates both primary and +// alternate SFiles entries, so isolated backends share that cache read-only. +func preparePackageBuilds(ctx *context, tasks []*packageBuildTask, verbose bool) error { + for _, task := range tasks { + if err := prePackageBuild(ctx, task, verbose); err != nil { + return err + } + if task.skip { + continue + } + coordinator, err := ctx.packageRequiresCoordinator(task) + if err != nil { + return err + } + task.isolated = !coordinator + _, patched := ctx.patches[task.pkg.PkgPath] + // typepatch.Merge still updates the shared patch.Types scope while a + // package is lowered. Keep patched packages isolated from the coordinator + // Program, but do not execute two such merges concurrently. + task.parallel = task.isolated && !patched + } + return nil +} + +// buildPackageGroup prepares the group serially, then runs eligible package +// backends with bounded parallelism. Archive creation and cache publication +// stay in the same package job as the backend that produced the object data. +func buildPackageGroup(ctx *context, tasks []*packageBuildTask, verbose bool) error { + if len(tasks) == 0 { + return nil + } + if err := preparePackageBuilds(ctx, tasks, verbose); err != nil { + return err + } + + isolated := make([]int, 0, len(tasks)) + // Complete coordinator and patched work before starting ordinary workers: + // patched lowering finalizes shared type scopes that other packages read. + for i, task := range tasks { + if task.skip { + continue + } + if task.parallel { + isolated = append(isolated, i) + continue + } + if err := buildPackage(ctx, task, verbose, task.isolated); err != nil { + return err + } + } + if err := runBoundedPackageJobs(ctx.buildConf.parallelism(), isolated, func(index int) error { + return buildPackage(ctx, tasks[index], verbose, true) + }); err != nil { + return err + } + return nil +} + +func buildPackage(ctx *context, task *packageBuildTask, verbose, isolated bool) error { + var err error + if isolated { + err = ctx.executeIsolatedPackage(task, verbose) + } else { + err = executePackageBuild(ctx, task, verbose) + } + if err != nil { + return err + } + return finalizePackageBuild(ctx, task, verbose) +} + +func runBoundedPackageJobs(parallelism int, indexes []int, run func(index int) error) error { + if len(indexes) == 0 { + return nil + } + workers := min(max(1, parallelism), len(indexes)) + jobs := make(chan int, len(indexes)) + errs := make([]error, len(indexes)) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for pos := range jobs { + errs[pos] = runPackageJob(indexes[pos], run) + } + }() + } + for pos := range indexes { + jobs <- pos + } + close(jobs) + wg.Wait() + for _, err := range errs { + if err != nil { + return err + } + } + return nil +} + +func runPackageJob(index int, run func(index int) error) (err error) { + defer func() { + if value := recover(); value != nil { + if recovered, ok := value.(error); ok { + err = recovered + } else { + err = fmt.Errorf("package job %d panicked: %v", index, value) + } + } + }() + return run(index) +} + +func (ctx *context) canUseIsolatedBackend() bool { + return ctx.mode != ModeGen && + ctx.buildConf.BuildMode == BuildModeExe && + ctx.buildConf.ModuleHook == nil +} + +func (ctx *context) packageRequiresCoordinator(task *packageBuildTask) (bool, error) { + if !ctx.canUseIsolatedBackend() { + return true, nil + } + usesPlan9, err := ctx.packageUsesPlan9Asm(task.pkg) + if err != nil { + return false, err + } + return usesPlan9, nil +} + +func (ctx *context) packageUsesPlan9Asm(pkg *aPackage) (bool, error) { + check := func(p *packages.Package) (bool, error) { + if p == nil { + return false, nil + } + sfiles, err := pkgSFiles(ctx, p) + if err != nil { + return false, err + } + return len(sfiles) != 0 && ctx.plan9asmEnabled(p.PkgPath), nil + } + if yes, err := check(pkg.Package); err != nil || yes { + return yes, err + } + if pkg.AltPkg != nil { + return check(pkg.AltPkg.Package) + } + return false, nil +} + +func (ctx *context) executeIsolatedPackage(task *packageBuildTask, verbose bool) error { + session := ctx.newBackendSession() + owned := true + defer func() { + if owned { + task.pkg.LPkg = nil + session.prog.Dispose() + } + }() + backendCtx := ctx.newBackendTask(session) + + if err := buildPkg(backendCtx, task.pkg, verbose); err != nil { + return err + } + if task.pkg.LPkg == nil { + return nil + } + owned = false + if task.needsRuntimeSignals() { + task.pkg.setNeedRuntimeOrPyInit(task.pkg.LPkg.NeedRuntime, task.pkg.LPkg.NeedPyInit) + } + // Linking still consumes live package state: method tables, globals, + // funcinfo/PCLN, C exports, and DCE source modules. Cache hits still rebuild + // that frontend module in the isolated Program, skip backend emission, and + // keep the module alive until every link has completed. + // + // LPkg retains the Program that owns its LLVM context. Ownership therefore + // moves to the coordinator on every success, not only when dead-code + // dropping is enabled. + return nil +} + +func (ctx *context) newBackendTask(session backendSession) *context { + // preparePackageBuilds populated every task's SFiles entry before workers start. + // Backend tasks share that map read-only; a frozen miss returns an error. + return &context{ + conf: ctx.conf, + progSSA: ctx.progSSA, + prog: session.prog, + dedup: ctx.dedup, + patches: ctx.patches, + callerTracking: ctx.callerTracking, + initial: ctx.initial, + pkgs: ctx.pkgs, + pkgByID: ctx.pkgByID, + mode: ctx.mode, + output: ctx.output, + passOpt: ctx.passOpt, + buildConf: ctx.buildConf, + crossCompile: ctx.crossCompile, + commands: ctx.commands, + frontendOptions: ctx.frontendOptions, + cTransformer: session.transformer, + sfilesCache: ctx.sfilesCache, + sfilesFrozen: true, + plan9asmReady: true, + plan9asmMode: ctx.plan9asmMode, + plan9asmPkgs: ctx.plan9asmPkgs, + } } -func packageBuildResultFor(task *packageBuildTask) packageBuildResult { - return packageBuildResult{ - needRuntime: task.pkg.NeedRt, - needPyInit: task.pkg.NeedPyInit, +func packageRuntimeNeeds(tasks []*packageBuildTask) (needRuntime, needPyInit bool) { + for _, task := range tasks { + needRuntime = needRuntime || task.pkg.NeedRt + needPyInit = needPyInit || task.pkg.NeedPyInit } + return } diff --git a/internal/build/package_build_test.go b/internal/build/package_build_test.go index 235e77e7cc..d1e56a49de 100644 --- a/internal/build/package_build_test.go +++ b/internal/build/package_build_test.go @@ -17,21 +17,27 @@ package build import ( + "errors" "go/ast" "go/parser" "go/token" "go/types" + "os" "path/filepath" + "runtime" "strings" + "sync/atomic" "testing" + "time" "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" "golang.org/x/tools/go/ssa" ) -func TestPackageBuildTaskAndResult(t *testing.T) { +func TestPackageBuildTask(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ PkgPath: "example.com/p", GoFiles: []string{"p.go"}, @@ -41,9 +47,9 @@ func TestPackageBuildTaskAndResult(t *testing.T) { if task.isDeclOnly() || task.isLinkOnly() || !task.hasSource() || task.isRuntime() || !task.needsRuntimeSignals() { t.Fatalf("unexpected normal package task: %+v", task) } - result := packageBuildResultFor(task) - if !result.needRuntime || !result.needPyInit { - t.Fatalf("unexpected package result: %+v", result) + needRuntime, needPyInit := packageRuntimeNeeds([]*packageBuildTask{task}) + if !needRuntime || !needPyInit { + t.Fatalf("package runtime needs = %v, %v; want true, true", needRuntime, needPyInit) } } @@ -102,41 +108,6 @@ var content string } } -func TestBuildOnePackageReturnsFrontendError(t *testing.T) { - t.Setenv(llgoBuildCache, "off") - fset := token.NewFileSet() - filename := filepath.Join(t.TempDir(), "p.go") - file, err := parser.ParseFile(fset, filename, `package p - -//go:embed missing.txt -var content string -`, parser.ParseComments) - if err != nil { - t.Fatal(err) - } - pkg := &aPackage{ - Package: &packages.Package{ - ID: "example.com/frontend-error", - PkgPath: "example.com/frontend-error", - GoFiles: []string{filename}, - Syntax: []*ast.File{file}, - Types: types.NewPackage("example.com/frontend-error", "p"), - }, - Manifest: "already fingerprinted", - Fingerprint: "frontend-error", - } - ctx := &context{ - conf: &packages.Config{Fset: fset}, - buildConf: &Config{}, - built: make(map[string]none), - } - - _, err = buildOnePackage(ctx, newPackageBuildTask(pkg), false) - if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { - t.Fatalf("buildOnePackage frontend error = %v", err) - } -} - func TestPrePackageBuildErrorsAndCacheHit(t *testing.T) { t.Run("fingerprint error", func(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ @@ -147,7 +118,6 @@ func TestPrePackageBuildErrorsAndCacheHit(t *testing.T) { }} ctx := &context{ buildConf: &Config{}, - built: make(map[string]none), llvmVersion: "test", } task := newPackageBuildTask(pkg) @@ -173,7 +143,7 @@ func TestPrePackageBuildErrorsAndCacheHit(t *testing.T) { Fingerprint: "cache-hit", CacheHit: true, } - ctx := &context{buildConf: &Config{}, built: make(map[string]none)} + ctx := &context{buildConf: &Config{}} task := newPackageBuildTask(pkg) err := prePackageBuild(ctx, task, true) if err != nil { @@ -185,24 +155,6 @@ func TestPrePackageBuildErrorsAndCacheHit(t *testing.T) { }) } -func TestBuildOnePackageSkipsAlreadyBuiltPackage(t *testing.T) { - pkg := &aPackage{Package: &packages.Package{ - ID: "example.com/already-built", - PkgPath: "example.com/already-built", - GoFiles: []string{"already.go"}, - Types: types.NewPackage("example.com/already-built", "already"), - }, NeedRt: true} - ctx := &context{built: map[string]none{pkg.ID: {}}} - - result, err := buildOnePackage(ctx, newPackageBuildTask(pkg), false) - if err != nil { - t.Fatal(err) - } - if !result.needRuntime { - t.Fatalf("build result = %+v, want runtime requirement preserved", result) - } -} - func TestPrePackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "unsafe", @@ -210,7 +162,7 @@ func TestPrePackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { Types: types.Unsafe, ExportFile: "stale.a", }} - ctx := &context{built: make(map[string]none)} + ctx := &context{} task := newPackageBuildTask(pkg) err := prePackageBuild(ctx, task, false) @@ -223,9 +175,6 @@ func TestPrePackageBuildSkipsDeclarationOnlyPackage(t *testing.T) { if pkg.ExportFile != "" { t.Fatalf("ExportFile = %q, want empty", pkg.ExportFile) } - if _, ok := ctx.built[pkg.ID]; !ok { - t.Fatal("declaration-only package was not recorded as built") - } } func TestPrePackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { @@ -236,7 +185,7 @@ func TestPrePackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { Types: types.NewPackage("example.com/linkonly", "linkonly"), ExportFile: "stale.a", }} - ctx := &context{buildConf: &Config{}, built: make(map[string]none)} + ctx := &context{buildConf: &Config{}} task := &packageBuildTask{pkg: pkg, kind: cl.PkgLinkExtern, kindParam: "-lexample"} err := prePackageBuild(ctx, task, false) if err != nil { @@ -250,19 +199,18 @@ func TestPrePackageBuildSkipsExternalLinkOnlyPackage(t *testing.T) { } } -func TestFinalizePackageBuildReturnsCachedResult(t *testing.T) { +func TestFinalizePackageBuildPreservesCachedRuntimeNeeds(t *testing.T) { pkg := &aPackage{Package: &packages.Package{ ID: "example.com/cached", PkgPath: "example.com/cached", Types: types.NewPackage("example.com/cached", "cached"), }, CacheHit: true, NeedPyInit: true} - result, err := finalizePackageBuild(&context{}, newPackageBuildTask(pkg), false) - if err != nil { + if err := finalizePackageBuild(&context{}, newPackageBuildTask(pkg), false); err != nil { t.Fatal(err) } - if !result.needPyInit { - t.Fatalf("build result = %+v, want Python initialization requirement preserved", result) + if !pkg.NeedPyInit { + t.Fatal("cached package lost Python initialization requirement") } } @@ -275,3 +223,369 @@ func TestBuildSSAPkgsEmptyAndNilEntries(t *testing.T) { pkg := prog.CreatePackage(types.NewPackage("example.com/ssa", "ssa"), nil, nil, true) buildSSAPkgs(ctx, []ssaBuildEntry{{pkg: pkg}, {pkg: pkg}}) } +func TestCanUseIsolatedBackend(t *testing.T) { + ctx := &context{ + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + } + if !ctx.canUseIsolatedBackend() { + t.Fatal("normal executable build should use isolated backends") + } + ctx.mode = ModeGen + if ctx.canUseIsolatedBackend() { + t.Fatal("generation mode should remain on the coordinator") + } + ctx.mode = ModeTest + if !ctx.canUseIsolatedBackend() { + t.Fatal("test mode should use isolated executable backends") + } + ctx.buildConf.BuildMode = BuildModeCShared + if ctx.canUseIsolatedBackend() { + t.Fatal("c-shared mode should remain on the coordinator") + } + ctx.buildConf.BuildMode = BuildModeExe + ctx.buildConf.ModuleHook = func(*aPackage) {} + if ctx.canUseIsolatedBackend() { + t.Fatal("module hooks should remain on the coordinator") + } +} + +func TestPreparePackageBuildsKeepsPatchesIsolatedAndSerial(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + patched := &aPackage{Package: &packages.Package{ + ID: "example.com/patched", + PkgPath: "example.com/patched", + }} + normal := &aPackage{Package: &packages.Package{ + ID: "example.com/normal", + PkgPath: "example.com/normal", + }} + ctx := &context{ + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + patches: cl.Patches{patched.PkgPath: {}}, + sfilesCache: make(map[string][]string), + } + patchedTask := &packageBuildTask{pkg: patched} + normalTask := &packageBuildTask{pkg: normal} + if err := preparePackageBuilds(ctx, []*packageBuildTask{patchedTask, normalTask}, false); err != nil { + t.Fatal(err) + } + if !patchedTask.isolated || patchedTask.parallel { + t.Fatalf("patched execution = isolated %v, parallel %v; want true, false", patchedTask.isolated, patchedTask.parallel) + } + if !normalTask.isolated || !normalTask.parallel { + t.Fatalf("normal execution = isolated %v, parallel %v; want true, true", normalTask.isolated, normalTask.parallel) + } +} + +func TestBuildPackageGroupReturnsPreparationError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + pkg := &aPackage{Package: &packages.Package{ + ID: "example.com/missing-source", + PkgPath: "example.com/missing-source", + GoFiles: []string{filepath.Join(t.TempDir(), "missing.go")}, + Types: types.NewPackage("example.com/missing-source", "missing"), + }} + ctx := &context{buildConf: &Config{}, llvmVersion: "test"} + err := buildPackageGroup(ctx, []*packageBuildTask{newPackageBuildTask(pkg)}, false) + if err == nil || !strings.Contains(err.Error(), "digest go files") { + t.Fatalf("build package group preparation error = %v", err) + } +} + +func TestPreparePackageBuildsReturnsCoordinatorReadError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + pkg := &aPackage{ + Package: &packages.Package{ + ID: "example.com/unprepared-asm", + PkgPath: "example.com/unprepared-asm", + Types: types.NewPackage("example.com/unprepared-asm", "unprepared"), + }, + Manifest: "prepared manifest", + Fingerprint: "prepared fingerprint", + } + ctx := &context{ + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + sfilesFrozen: true, + } + err := preparePackageBuilds(ctx, []*packageBuildTask{{pkg: pkg}}, false) + if err == nil || !strings.Contains(err.Error(), "were not prepared") { + t.Fatalf("prepare package coordinator read error = %v", err) + } +} + +func invalidEmbedPackage(t *testing.T) (*token.FileSet, *aPackage) { + t.Helper() + fset := token.NewFileSet() + filename := filepath.Join(t.TempDir(), "p.go") + file, err := parser.ParseFile(fset, filename, `package p + +//go:embed missing.txt +var content string +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + return fset, &aPackage{ + Package: &packages.Package{ + ID: "example.com/invalid-embed", + PkgPath: "example.com/invalid-embed", + GoFiles: []string{filename}, + Syntax: []*ast.File{file}, + Types: types.NewPackage("example.com/invalid-embed", "p"), + }, + Manifest: "prepared manifest", + Fingerprint: "prepared fingerprint", + } +} + +func TestBuildPackageGroupReturnsCoordinatorBuildError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + fset, pkg := invalidEmbedPackage(t) + ctx := &context{ + conf: &packages.Config{Fset: fset}, + mode: ModeGen, + buildConf: &Config{}, + } + err := buildPackageGroup(ctx, []*packageBuildTask{newPackageBuildTask(pkg)}, false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("coordinator build error = %v", err) + } +} + +func TestBuildPackageGroupReturnsParallelBuildError(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + fset, pkg := invalidEmbedPackage(t) + coordinator := llssa.NewProgram(&llssa.Target{GOOS: runtime.GOOS, GOARCH: runtime.GOARCH}) + defer coordinator.Dispose() + ctx := &context{ + conf: &packages.Config{Fset: fset}, + prog: coordinator, + mode: ModeBuild, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + }, + } + err := buildPackageGroup(ctx, []*packageBuildTask{newPackageBuildTask(pkg)}, false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("parallel build error = %v", err) + } +} + +func TestBuildAllPkgsReturnsPackageGroupErrors(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + newContext := func(fset *token.FileSet) *context { + return &context{ + conf: &packages.Config{Fset: fset}, + mode: ModeGen, + buildConf: &Config{}, + } + } + + t.Run("ordinary package", func(t *testing.T) { + fset, pkg := invalidEmbedPackage(t) + _, err := buildAllPkgs(newContext(fset), []*aPackage{pkg}, false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("ordinary package group error = %v", err) + } + }) + + t.Run("runtime package", func(t *testing.T) { + fset, pkg := invalidEmbedPackage(t) + pkg.ID = env.LLGoRuntimePkg + pkg.PkgPath = env.LLGoRuntimePkg + pkg.Types = types.NewPackage(env.LLGoRuntimePkg, "runtime") + _, err := buildAllPkgs(newContext(fset), []*aPackage{pkg}, false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("runtime package group error = %v", err) + } + }) +} + +func TestExecuteIsolatedPackageReleasesProgramWithoutModule(t *testing.T) { + coordinator := llssa.NewProgram(&llssa.Target{GOOS: runtime.GOOS, GOARCH: runtime.GOARCH}) + defer coordinator.Dispose() + ctx := &context{ + prog: coordinator, + buildConf: &Config{ + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + }, + } + + t.Run("frontend error", func(t *testing.T) { + fset, pkg := invalidEmbedPackage(t) + ctx.conf = &packages.Config{Fset: fset} + task := newPackageBuildTask(pkg) + err := ctx.executeIsolatedPackage(task, false) + if err == nil || !strings.Contains(err.Error(), "only allowed in Go files that import") { + t.Fatalf("isolated frontend error = %v", err) + } + if pkg.LPkg != nil { + t.Fatal("isolated frontend error retained LPkg") + } + }) + + t.Run("skipped package", func(t *testing.T) { + pkg := &aPackage{Package: &packages.Package{ID: "unsafe", PkgPath: "unsafe", Types: types.Unsafe}} + if err := ctx.executeIsolatedPackage(newPackageBuildTask(pkg), false); err != nil { + t.Fatal(err) + } + if pkg.LPkg != nil { + t.Fatal("skipped isolated package retained LPkg") + } + }) +} + +func TestPkgSFilesRejectsUnpreparedBackendRead(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "asm.s"), []byte("TEXT ·f(SB),$0-0\n\tRET\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx := &context{ + sfilesCache: make(map[string][]string), + sfilesFrozen: true, + } + _, err := pkgSFiles(ctx, &packages.Package{ + ID: "example.com/asm", + PkgPath: "example.com/asm", + Dir: dir, + }) + if err == nil { + t.Fatal("expected frozen SFiles cache to reject an unprepared package") + } +} + +func TestRunBoundedPackageJobs(t *testing.T) { + started := make(chan int, 4) + release := make(chan struct{}) + done := make(chan error, 1) + var active atomic.Int32 + var maximum atomic.Int32 + go func() { + done <- runBoundedPackageJobs(2, []int{0, 1, 2, 3}, func(index int) error { + current := active.Add(1) + for { + old := maximum.Load() + if current <= old || maximum.CompareAndSwap(old, current) { + break + } + } + started <- index + <-release + active.Add(-1) + return nil + }) + }() + + for range 2 { + select { + case <-started: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("two package workers did not start concurrently") + } + } + if got := maximum.Load(); got != 2 { + close(release) + t.Fatalf("maximum concurrent jobs = %d, want 2", got) + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if got := maximum.Load(); got != 2 { + t.Fatalf("maximum concurrent jobs = %d after completion, want 2", got) + } +} + +func TestRunBoundedPackageJobsReturnsFirstOrderedError(t *testing.T) { + first := errors.New("first") + second := errors.New("second") + err := runBoundedPackageJobs(3, []int{3, 1, 2}, func(index int) error { + switch index { + case 3: + return first + case 1: + return second + default: + return nil + } + }) + if !errors.Is(err, first) { + t.Fatalf("error = %v, want first submitted error", err) + } +} + +func TestRunBoundedPackageJobsConvertsPanicToError(t *testing.T) { + boom := errors.New("boom") + err := runBoundedPackageJobs(2, []int{0, 1}, func(index int) error { + if index == 0 { + panic(boom) + } + return nil + }) + if !errors.Is(err, boom) { + t.Fatalf("error = %v, want recovered panic %v", err, boom) + } + err = runPackageJob(7, func(int) error { panic("boom") }) + if err == nil || !strings.Contains(err.Error(), "package job 7 panicked: boom") { + t.Fatalf("non-error panic = %v", err) + } +} + +func TestBuildPackageGroupEmpty(t *testing.T) { + ctx := &context{buildConf: &Config{}} + if err := buildPackageGroup(ctx, nil, false); err != nil { + t.Fatalf("empty package group = %v", err) + } +} + +func TestNewBackendTaskUsesPackageLocalState(t *testing.T) { + coordinator := &context{ + conf: &packages.Config{}, + mode: ModeBuild, + buildConf: &Config{BuildMode: BuildModeExe}, + commands: commandEnv{dir: t.TempDir()}, + frontendOptions: cl.Options{Debug: true}, + sfilesCache: map[string][]string{"example.com/p": {"asm.s"}}, + plan9asmReady: true, + plan9asmMode: plan9asmEnvSelected, + plan9asmPkgs: map[string]bool{"example.com/p": true}, + } + + task := coordinator.newBackendTask(backendSession{}) + if task == coordinator { + t.Fatal("backend task aliases coordinator") + } + if task.buildConf != coordinator.buildConf || task.conf != coordinator.conf { + t.Fatal("backend task did not retain immutable build inputs") + } + if !task.sfilesFrozen || !task.plan9asmReady || task.plan9asmMode != plan9asmEnvSelected { + t.Fatalf("backend task state = %+v", task) + } + if !task.frontendOptions.Debug || task.commands.dir != coordinator.commands.dir { + t.Fatal("backend task lost invocation settings") + } + if task.plan9asmPkgs["example.com/p"] != coordinator.plan9asmPkgs["example.com/p"] { + t.Fatal("backend task lost prepared Plan9 package policy") + } +} + +func TestPackageSchedulingHandlesNonBackendPackages(t *testing.T) { + task := &packageBuildTask{pkg: &aPackage{}} + ctx := &context{mode: ModeGen, buildConf: &Config{BuildMode: BuildModeExe}} + serial, err := ctx.packageRequiresCoordinator(task) + if err != nil || !serial { + t.Fatalf("generation package coordinator = %v, %v; want true, nil", serial, err) + } + + usesPlan9, err := (&context{}).packageUsesPlan9Asm(task.pkg) + if err != nil || usesPlan9 { + t.Fatalf("nil package Plan9 asm = %v, %v; want false, nil", usesPlan9, err) + } +} diff --git a/internal/build/plan9asm.go b/internal/build/plan9asm.go index 30077f5f72..e94eec3998 100644 --- a/internal/build/plan9asm.go +++ b/internal/build/plan9asm.go @@ -306,19 +306,22 @@ func cabiSkipFuncsForPlan9Asm(ctx *context, pkgPath string, mod gllvm.Module) [] } func (ctx *context) plan9asmEnabled(pkgPath string) bool { - ctx.plan9asmOnce.Do(func() { - cfg := parsePlan9AsmPkgsEnv(Plan9ASMPkgs()) - ctx.plan9asmMode = cfg.mode - switch cfg.mode { - case plan9asmEnvSelected: - ctx.plan9asmPkgs = make(map[string]bool, len(cfg.pkgs)) - for p := range cfg.pkgs { - ctx.plan9asmPkgs[p] = true + if !ctx.plan9asmReady { + ctx.plan9asmOnce.Do(func() { + cfg := parsePlan9AsmPkgsEnv(Plan9ASMPkgs()) + ctx.plan9asmMode = cfg.mode + switch cfg.mode { + case plan9asmEnvSelected: + ctx.plan9asmPkgs = make(map[string]bool, len(cfg.pkgs)) + for p := range cfg.pkgs { + ctx.plan9asmPkgs[p] = true + } + default: + ctx.plan9asmPkgs = make(map[string]bool) } - default: - ctx.plan9asmPkgs = make(map[string]bool) - } - }) + ctx.plan9asmReady = true + }) + } switch ctx.plan9asmMode { case plan9asmEnvAll: @@ -379,26 +382,32 @@ func pkgSFiles(ctx *context, pkg *packages.Package) ([]string, error) { if pkg == nil || pkg.PkgPath == "" { return nil, nil } + if ctx.sfilesCache == nil { + if ctx.sfilesFrozen { + return nil, fmt.Errorf("package %s assembly files were not prepared before backend execution", pkg.PkgPath) + } + ctx.sfilesCache = make(map[string][]string) + } + if v, ok := ctx.sfilesCache[pkg.ID]; ok { + return v, nil + } // Some unit tests construct synthetic packages that are not loadable via // `go list` (PkgPath not in any module, and Dir/Standard/Goroot unset). // In that case, treat the package as having no selected .s files. if pkg.Dir == "" { + ctx.sfilesCache[pkg.ID] = nil return nil, nil } // Fast path: if directory has no .s/.S at all, skip `go list`. - if pkg.Dir != "" { - if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.s")); len(ss) == 0 { - if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.S")); len(ss) == 0 { - return nil, nil - } + if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.s")); len(ss) == 0 { + if ss, _ := filepath.Glob(filepath.Join(pkg.Dir, "*.S")); len(ss) == 0 { + ctx.sfilesCache[pkg.ID] = nil + return nil, nil } } - if ctx.sfilesCache == nil { - ctx.sfilesCache = make(map[string][]string) - } - if v, ok := ctx.sfilesCache[pkg.ID]; ok { - return v, nil + if ctx.sfilesFrozen { + return nil, fmt.Errorf("package %s assembly files were not prepared before backend execution", pkg.PkgPath) } args := []string{"list", "-json"} diff --git a/internal/build/plan9asm_sfiles_test.go b/internal/build/plan9asm_sfiles_test.go index 400c8c4d7a..c2eba52474 100644 --- a/internal/build/plan9asm_sfiles_test.go +++ b/internal/build/plan9asm_sfiles_test.go @@ -109,3 +109,28 @@ printf '{"Dir":"%s","SFiles":["asm_amd64.s"]}\n' "$PACKAGE_DIR" t.Fatalf("pkgSFiles = %v, want [%s]", got, sfile) } } + +func TestPlan9AsmEnabledInitializesSelectedPackages(t *testing.T) { + t.Setenv(llgoPlan9ASMPkgs, "example.com/first, example.com/second") + ctx := &context{buildConf: &Config{}} + if !ctx.plan9asmEnabled("example.com/first") { + t.Fatal("selected Plan 9 assembly package was disabled") + } + if ctx.plan9asmEnabled("example.com/other") { + t.Fatal("unselected Plan 9 assembly package was enabled") + } + if !ctx.plan9asmReady || ctx.plan9asmMode != plan9asmEnvSelected || !ctx.plan9asmPkgs["example.com/second"] { + t.Fatalf("prepared Plan 9 assembly policy = mode %v, packages %v", ctx.plan9asmMode, ctx.plan9asmPkgs) + } +} + +func TestPkgSFilesRejectsNilFrozenCache(t *testing.T) { + ctx := &context{sfilesFrozen: true} + _, err := pkgSFiles(ctx, &packages.Package{ + ID: "example.com/unprepared", + PkgPath: "example.com/unprepared", + }) + if err == nil { + t.Fatal("nil frozen SFiles cache accepted an unprepared package") + } +} diff --git a/internal/dcepass/dcepass.go b/internal/dcepass/dcepass.go index db37db6725..ccbbd8c157 100644 --- a/internal/dcepass/dcepass.go +++ b/internal/dcepass/dcepass.go @@ -41,10 +41,15 @@ func EmitStrongTypeOverrides(dst llvm.Module, srcMods []llvm.Module, liveSlots m type overrideEmitter struct { dst llvm.Module values map[llvm.Value]llvm.Value + types map[llvm.Type]llvm.Type } func newOverrideEmitter(dst llvm.Module) *overrideEmitter { - return &overrideEmitter{dst: dst, values: make(map[llvm.Value]llvm.Value)} + return &overrideEmitter{ + dst: dst, + values: make(map[llvm.Value]llvm.Value), + types: make(map[llvm.Type]llvm.Type), + } } func (e *overrideEmitter) emitTypeOverride(srcType, methodsVal llvm.Value, elemTy llvm.Type, keepIdx map[int]bool, verbose bool) { @@ -59,6 +64,7 @@ func (e *overrideEmitter) emitTypeOverride(srcType, methodsVal llvm.Value, elemT } unreachableMethod := e.unreachableMethod() + dstElemTy := e.cloneType(elemTy) methods := make([]llvm.Value, methodsVal.OperandsCount()) for i := range methods { orig := methodsVal.Operand(i) @@ -71,16 +77,16 @@ func (e *overrideEmitter) emitTypeOverride(srcType, methodsVal llvm.Value, elemT } name := e.cloneConst(orig.Operand(0)) mtype := e.cloneConst(orig.Operand(1)) - methods[i] = llvm.ConstNamedStruct(elemTy, []llvm.Value{ + methods[i] = llvm.ConstNamedStruct(dstElemTy, []llvm.Value{ name, mtype, unreachableMethod, unreachableMethod, }) } - fields[fieldCount-1] = llvm.ConstArray(elemTy, methods) + fields[fieldCount-1] = llvm.ConstArray(dstElemTy, methods) - dstType.SetInitializer(constStructOfType(init.Type(), fields)) + dstType.SetInitializer(constStructOfType(e.cloneType(init.Type()), fields)) dstType.SetGlobalConstant(true) dstType.SetLinkage(llvm.ExternalLinkage) copyGlobalAttrs(dstType, srcType) @@ -99,7 +105,7 @@ func (e *overrideEmitter) ensureOverrideGlobal(src llvm.Value) llvm.Value { name := src.Name() dst := e.dst.NamedGlobal(name) if dst.IsNil() { - dst = llvm.AddGlobal(e.dst, src.GlobalValueType(), name) + dst = llvm.AddGlobal(e.dst, e.cloneType(src.GlobalValueType()), name) } e.values[src] = dst return dst @@ -112,12 +118,142 @@ func (e *overrideEmitter) cloneConst(v llvm.Value) llvm.Value { if gv := v.IsAGlobalValue(); !gv.IsNil() { return e.cloneGlobalValue(gv) } + dstTy := e.cloneType(v.Type()) + if v.IsNull() || !v.IsAConstantAggregateZero().IsNil() { + return llvm.ConstNull(dstTy) + } + if !v.IsAUndefValue().IsNil() { + return llvm.Undef(dstTy) + } + if !v.IsAConstantInt().IsNil() { + return llvm.ConstInt(dstTy, v.ZExtValue(), false) + } + if !v.IsAConstantFP().IsNil() { + value, _ := v.DoubleValue() + return llvm.ConstFloat(dstTy, value) + } + if v.IsConstantString() { + return e.dst.Context().ConstString(v.ConstGetAsString(), false) + } if !v.IsAConstantStruct().IsNil() { - clone := constStructOfType(v.Type(), e.cloneOperands(v)) + clone := constStructOfType(dstTy, e.cloneOperands(v)) + e.values[v] = clone + return clone + } + if !v.IsAConstantArray().IsNil() { + clone := llvm.ConstArray(dstTy.ElementType(), e.cloneOperands(v)) + e.values[v] = clone + return clone + } + if !v.IsAConstantVector().IsNil() { + clone := llvm.ConstVector(e.cloneOperands(v), false) e.values[v] = clone return clone } - return v + if !v.IsAConstantExpr().IsNil() { + return e.cloneConstExpr(v, dstTy) + } + panic(fmt.Sprintf("dcepass: unsupported constant %s", v.String())) +} + +func (e *overrideEmitter) cloneConstExpr(v llvm.Value, dstTy llvm.Type) llvm.Value { + ops := e.cloneOperands(v) + var clone llvm.Value + switch v.Opcode() { + case llvm.GetElementPtr: + clone = llvm.ConstGEP(e.cloneType(v.GEPSourceElementType()), ops[0], ops[1:]) + case llvm.BitCast: + clone = llvm.ConstBitCast(ops[0], dstTy) + case llvm.IntToPtr: + clone = llvm.ConstIntToPtr(ops[0], dstTy) + case llvm.PtrToInt: + clone = llvm.ConstPtrToInt(ops[0], dstTy) + case llvm.Trunc: + clone = llvm.ConstTrunc(ops[0], dstTy) + case llvm.Add: + clone = llvm.ConstAdd(ops[0], ops[1]) + case llvm.Sub: + clone = llvm.ConstSub(ops[0], ops[1]) + case llvm.Xor: + clone = llvm.ConstXor(ops[0], ops[1]) + default: + panic(fmt.Sprintf("dcepass: unsupported constant expression %s", v.String())) + } + e.values[v] = clone + return clone +} + +// cloneType re-interns an LLVM type in the destination module's Context. +// Returning a source-context type is invalid even when its printed spelling is +// identical; LLVM otherwise permits malformed mixed-context constants that +// fail verification and can crash while either Context is disposed. +func (e *overrideEmitter) cloneType(src llvm.Type) llvm.Type { + if dst, ok := e.types[src]; ok { + return dst + } + ctx := e.dst.Context() + var dst llvm.Type + switch src.TypeKind() { + case llvm.VoidTypeKind: + dst = ctx.VoidType() + case llvm.FloatTypeKind: + dst = ctx.FloatType() + case llvm.DoubleTypeKind: + dst = ctx.DoubleType() + case llvm.X86_FP80TypeKind: + dst = ctx.X86FP80Type() + case llvm.FP128TypeKind: + dst = ctx.FP128Type() + case llvm.PPC_FP128TypeKind: + dst = ctx.PPCFP128Type() + case llvm.LabelTypeKind: + dst = ctx.LabelType() + case llvm.IntegerTypeKind: + dst = ctx.IntType(src.IntTypeWidth()) + case llvm.FunctionTypeKind: + params := src.ParamTypes() + for i := range params { + params[i] = e.cloneType(params[i]) + } + dst = llvm.FunctionType(e.cloneType(src.ReturnType()), params, src.IsFunctionVarArg()) + case llvm.StructTypeKind: + name := src.StructName() + if name != "" { + dst = e.dst.GetTypeByName(name) + if dst.IsNil() { + dst = ctx.StructCreateNamed(name) + } + e.types[src] = dst + if dst.StructElementTypesCount() == 0 && src.StructElementTypesCount() != 0 { + dst.StructSetBody(e.cloneTypes(src.StructElementTypes()), src.IsStructPacked()) + } + return dst + } + dst = ctx.StructType(e.cloneTypes(src.StructElementTypes()), src.IsStructPacked()) + case llvm.ArrayTypeKind: + dst = llvm.ArrayType(e.cloneType(src.ElementType()), src.ArrayLength()) + case llvm.PointerTypeKind: + // LLVM uses opaque pointers; the element type only selects the Context. + dst = llvm.PointerType(ctx.Int8Type(), src.PointerAddressSpace()) + case llvm.VectorTypeKind: + dst = llvm.VectorType(e.cloneType(src.ElementType()), src.VectorSize()) + case llvm.MetadataTypeKind: + dst = ctx.MetadataType() + case llvm.TokenTypeKind: + dst = ctx.TokenType() + default: + panic(fmt.Sprintf("dcepass: unsupported LLVM type kind %d", src.TypeKind())) + } + e.types[src] = dst + return dst +} + +func (e *overrideEmitter) cloneTypes(src []llvm.Type) []llvm.Type { + dst := make([]llvm.Type, len(src)) + for i := range src { + dst[i] = e.cloneType(src[i]) + } + return dst } func (e *overrideEmitter) cloneOperands(v llvm.Value) []llvm.Value { @@ -136,7 +272,7 @@ func (e *overrideEmitter) cloneGlobalValue(v llvm.Value) llvm.Value { if fn := v.IsAFunction(); !fn.IsNil() { dstFn := e.dst.NamedFunction(fn.Name()) if dstFn.IsNil() { - dstFn = llvm.AddFunction(e.dst, fn.Name(), fn.GlobalValueType()) + dstFn = llvm.AddFunction(e.dst, fn.Name(), e.cloneType(fn.GlobalValueType())) } e.values[v] = dstFn return dstFn @@ -155,14 +291,14 @@ func (e *overrideEmitter) cloneGlobalVariable(src llvm.Value) llvm.Value { if name != "" && !isLocalLinkage(src.Linkage()) { dst := e.dst.NamedGlobal(name) if dst.IsNil() { - dst = llvm.AddGlobal(e.dst, src.GlobalValueType(), name) + dst = llvm.AddGlobal(e.dst, e.cloneType(src.GlobalValueType()), name) dst.SetLinkage(llvm.ExternalLinkage) } e.values[src] = dst return dst } - dst := llvm.AddGlobal(e.dst, src.GlobalValueType(), "") + dst := llvm.AddGlobal(e.dst, e.cloneType(src.GlobalValueType()), "") e.values[src] = dst copyGlobalAttrs(dst, src) dst.SetLinkage(src.Linkage()) @@ -212,8 +348,9 @@ func isLocalLinkage(linkage llvm.Linkage) bool { } func constStructOfType(typ llvm.Type, fields []llvm.Value) llvm.Value { - if typ.StructName() != "" { - return llvm.ConstNamedStruct(typ, fields) - } - return llvm.ConstStruct(fields, typ.IsStructPacked()) + // LLVMConstNamedStruct accepts both identified and literal struct types. Use + // the exact cloned type: LLVMConstStruct(InContext) may manufacture another + // structurally identical literal type that is not pointer-identical to a + // global's declared value type, which the verifier correctly rejects. + return llvm.ConstNamedStruct(typ, fields) } diff --git a/internal/dcepass/dcepass_test.go b/internal/dcepass/dcepass_test.go index c275804e30..d05d1145de 100644 --- a/internal/dcepass/dcepass_test.go +++ b/internal/dcepass/dcepass_test.go @@ -1,8 +1,10 @@ package dcepass import ( + "fmt" "os" "path/filepath" + "strings" "testing" qtest "github.com/qiniu/x/test" @@ -29,15 +31,25 @@ func TestEmitStrongTypeOverrides(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := llvm.NewContext() - defer ctx.Dispose() + srcCtx := llvm.NewContext() + defer srcCtx.Dispose() + dstCtx := llvm.NewContext() + defer dstCtx.Dispose() dir := filepath.Join("testdata", tt.name) - src := parseModule(t, &ctx, filepath.Join(dir, "in.ll")) + src := parseModule(t, &srcCtx, filepath.Join(dir, "in.ll")) defer src.Dispose() - dst := ctx.NewModule("dst") + dst := dstCtx.NewModule("dst") defer dst.Dispose() EmitStrongTypeOverrides(dst, []llvm.Module{src}, tt.liveSlots, true) + for global := dst.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if init := global.Initializer(); !init.IsNil() && global.GlobalValueType().C != init.Type().C { + t.Errorf("global %s type %s does not match initializer type %s", global.Name(), global.GlobalValueType(), init.Type()) + } + } + if err := llvm.VerifyModule(dst, llvm.ReturnStatusAction); err != nil { + t.Fatalf("cross-context override is invalid: %v\n%s", err, dst.String()) + } want, err := os.ReadFile(filepath.Join(dir, "expect.ll")) if err != nil { t.Fatal(err) @@ -98,6 +110,207 @@ func TestMethodArray(t *testing.T) { } } +func TestCloneTypesAcrossContexts(t *testing.T) { + srcCtx := llvm.NewContext() + defer srcCtx.Dispose() + dstCtx := llvm.NewContext() + defer dstCtx.Dispose() + dst := dstCtx.NewModule("dst") + defer dst.Dispose() + + emitter := newOverrideEmitter(dst) + named := srcCtx.StructCreateNamed("named") + named.StructSetBody([]llvm.Type{srcCtx.Int32Type()}, false) + existing := srcCtx.StructCreateNamed("existing") + existing.StructSetBody([]llvm.Type{srcCtx.Int32Type()}, false) + dstExisting := dstCtx.StructCreateNamed("existing") + dstExisting.StructSetBody([]llvm.Type{dstCtx.Int64Type()}, false) + + types := []llvm.Type{ + srcCtx.VoidType(), + srcCtx.FloatType(), + srcCtx.DoubleType(), + srcCtx.X86FP80Type(), + srcCtx.FP128Type(), + srcCtx.PPCFP128Type(), + srcCtx.LabelType(), + srcCtx.IntType(17), + llvm.FunctionType(srcCtx.VoidType(), []llvm.Type{srcCtx.Int32Type()}, true), + named, + existing, + srcCtx.StructType([]llvm.Type{srcCtx.Int8Type(), srcCtx.Int16Type()}, true), + llvm.ArrayType(srcCtx.Int32Type(), 3), + llvm.PointerType(srcCtx.Int8Type(), 5), + llvm.VectorType(srcCtx.Int16Type(), 4), + srcCtx.MetadataType(), + srcCtx.TokenType(), + } + for _, src := range types { + got := emitter.cloneType(src) + if got.IsNil() { + t.Fatalf("cloneType(%s) returned nil", src) + } + if got.TypeKind() != src.TypeKind() { + t.Fatalf("cloneType(%s) kind = %v, want %v", src, got.TypeKind(), src.TypeKind()) + } + if cached := emitter.cloneType(src); cached.C != got.C { + t.Fatalf("cloneType(%s) did not reuse the destination type", src) + } + } + + clonedNamed := emitter.cloneType(named) + if clonedNamed.C == named.C || clonedNamed.StructElementTypesCount() != 1 { + t.Fatalf("identified struct was not recreated in the destination context: %s", clonedNamed) + } + if got := emitter.cloneType(existing); got.C != dstExisting.C { + t.Fatalf("existing destination type was not reused: got %s, want %s", got, dstExisting) + } + if got := dstExisting.StructElementTypes()[0].IntTypeWidth(); got != 64 { + t.Fatalf("existing destination type body was overwritten: width = %d, want 64", got) + } +} + +func TestCloneConstantsAcrossContexts(t *testing.T) { + srcCtx := llvm.NewContext() + defer srcCtx.Dispose() + dstCtx := llvm.NewContext() + defer dstCtx.Dispose() + src := srcCtx.NewModule("src") + defer src.Dispose() + dst := dstCtx.NewModule("dst") + defer dst.Dispose() + + emitter := newOverrideEmitter(dst) + i32 := srcCtx.Int32Type() + i64 := srcCtx.Int64Type() + zero := llvm.ConstInt(i32, 0, false) + one := llvm.ConstInt(i32, 1, false) + arrayTy := llvm.ArrayType(i32, 2) + array := llvm.AddGlobal(src, arrayTy, "array") + array.SetInitializer(llvm.ConstArray(i32, []llvm.Value{zero, one})) + ptrTy := llvm.PointerType(srcCtx.Int8Type(), 0) + local := llvm.AddGlobal(src, i32, "local") + local.SetLinkage(llvm.InternalLinkage) + local.SetGlobalConstant(true) + local.SetAlignment(8) + local.SetInitializer(one) + fn := llvm.AddFunction(src, "function", llvm.FunctionType(srcCtx.VoidType(), nil, false)) + + structTy := srcCtx.StructType([]llvm.Type{i32, srcCtx.DoubleType()}, false) + constants := []llvm.Value{ + llvm.ConstNull(i32), + llvm.Undef(i32), + one, + llvm.ConstFloat(srcCtx.DoubleType(), 1.5), + srcCtx.ConstString("llgo", false), + llvm.ConstNamedStruct(structTy, []llvm.Value{one, llvm.ConstFloat(srcCtx.DoubleType(), 2.5)}), + llvm.ConstArray(ptrTy, []llvm.Value{array, array}), + llvm.ConstVector([]llvm.Value{array, array}, false), + fn, + array, + local, + } + for i, source := range constants { + cloned := emitter.cloneConst(source) + if cloned.IsNil() { + t.Fatalf("cloneConst(%s) returned nil", source) + } + if cached := emitter.cloneConst(source); cached.C != cloned.C && !source.IsAConstantInt().IsNil() { + t.Fatalf("cloneConst(%s) did not reuse the destination value", source) + } + if source.IsAGlobalValue().IsNil() { + global := llvm.AddGlobal(dst, cloned.Type(), fmt.Sprintf("clone.%d", i)) + global.SetInitializer(cloned) + } + } + + ptrInt := llvm.ConstPtrToInt(array, i64) + expressions := []llvm.Value{ + llvm.ConstGEP(arrayTy, array, []llvm.Value{zero, one}), + llvm.ConstIntToPtr(llvm.ConstInt(i64, 16, false), llvm.PointerType(srcCtx.Int8Type(), 0)), + ptrInt, + llvm.ConstTrunc(ptrInt, i32), + llvm.ConstAdd(ptrInt, llvm.ConstInt(i64, 1, false)), + llvm.ConstSub(ptrInt, llvm.ConstInt(i64, 1, false)), + llvm.ConstXor(ptrInt, llvm.ConstInt(i64, 1, false)), + } + for i, source := range expressions { + cloned := emitter.cloneConst(source) + global := llvm.AddGlobal(dst, cloned.Type(), fmt.Sprintf("expr.%d", i)) + global.SetInitializer(cloned) + } + + if err := llvm.VerifyModule(dst, llvm.ReturnStatusAction); err != nil { + t.Fatalf("cloned constants cross LLVM Contexts: %v\n%s", err, dst.String()) + } +} + +func TestCloneConstantsRejectsUnsupportedLLVMForms(t *testing.T) { + srcCtx := llvm.NewContext() + defer srcCtx.Dispose() + dstCtx := llvm.NewContext() + defer dstCtx.Dispose() + + path := filepath.Join(t.TempDir(), "constants.ll") + ir := ` +@target = external global i8 +@bitcast_expr = global double bitcast (i64 ptrtoint (ptr @target to i64) to double) +@trunc_expr = global i32 trunc (i64 add (i64 ptrtoint (ptr @target to i64), i64 1) to i32) +@unsupported_expr = global i64 mul (i64 ptrtoint (ptr @target to i64), i64 2) +@block_addr = global ptr blockaddress(@block_target, %entry) + +define void @block_target() { +entry: + ret void +} + +declare half @unsupported_type() +` + if err := os.WriteFile(path, []byte(ir), 0o644); err != nil { + t.Fatal(err) + } + src := parseModule(t, &srcCtx, path) + defer src.Dispose() + dst := dstCtx.NewModule("dst") + defer dst.Dispose() + emitter := newOverrideEmitter(dst) + + for _, name := range []string{"bitcast_expr", "trunc_expr"} { + value := src.NamedGlobal(name).Initializer() + clone := emitter.cloneConst(value) + if clone.IsNil() { + t.Fatalf("cloneConst(%s) returned nil", name) + } + global := llvm.AddGlobal(dst, clone.Type(), "clone."+name) + global.SetInitializer(clone) + } + if err := llvm.VerifyModule(dst, llvm.ReturnStatusAction); err != nil { + t.Fatalf("cloned bitcast/trunc expressions are invalid: %v\n%s", err, dst.String()) + } + + assertPanic := func(name, want string, run func()) { + t.Helper() + t.Run(name, func(t *testing.T) { + defer func() { + got := recover() + if got == nil || !strings.Contains(fmt.Sprint(got), want) { + t.Fatalf("panic = %v, want substring %q", got, want) + } + }() + run() + }) + } + assertPanic("constant", "unsupported constant", func() { + emitter.cloneConst(src.NamedGlobal("block_addr").Initializer()) + }) + assertPanic("constant expression", "unsupported constant expression", func() { + emitter.cloneConst(src.NamedGlobal("unsupported_expr").Initializer()) + }) + assertPanic("type", "unsupported LLVM type kind", func() { + emitter.cloneType(src.NamedFunction("unsupported_type").GlobalValueType().ReturnType()) + }) +} + func parseModule(t *testing.T, ctx *llvm.Context, path string) llvm.Module { t.Helper() buf, err := llvm.NewMemoryBufferFromFile(path) diff --git a/internal/dcepass/testdata/method_slots/expect.ll b/internal/dcepass/testdata/method_slots/expect.ll index f8aa19371a..6587cb59cb 100644 --- a/internal/dcepass/testdata/method_slots/expect.ll +++ b/internal/dcepass/testdata/method_slots/expect.ll @@ -9,7 +9,7 @@ source_filename = "dst" %"github.com/goplus/llgo/runtime/abi.Method" = type { %runtime.String, ptr, ptr, ptr } %"runtime/abi.PtrType" = type { %"runtime/abi.Type", ptr } %"runtime/abi.FuncType" = type { %"runtime/abi.Type", %runtime.Slice, %runtime.Slice } -%Task = type {} +%Task = type opaque @_llgo_main.Task = constant { %"runtime/abi.StructType", %"runtime/abi.UncommonType", [2 x %"github.com/goplus/llgo/runtime/abi.Method"] } { %"runtime/abi.StructType" { %"runtime/abi.Type" { i64 0, i64 0, i32 1, i8 13, i8 1, i8 1, i8 25, { ptr, ptr } { ptr @memequal0, ptr @_llgo_main.Task }, ptr null, %runtime.String { ptr @0, i64 9 }, ptr @"*_llgo_main.Task" }, %runtime.String zeroinitializer, %runtime.Slice zeroinitializer }, %"runtime/abi.UncommonType" { %runtime.String { ptr @1, i64 4 }, i16 2, i16 2, i32 24 }, [2 x %"github.com/goplus/llgo/runtime/abi.Method"] [%"github.com/goplus/llgo/runtime/abi.Method" { %runtime.String { ptr @2, i64 4 }, ptr @"_llgo_func$run", ptr @"github.com/goplus/llgo/runtime/internal/runtime.unreachableMethod", ptr @"github.com/goplus/llgo/runtime/internal/runtime.unreachableMethod" }, %"github.com/goplus/llgo/runtime/abi.Method" { %runtime.String { ptr @3, i64 3 }, ptr @"_llgo_func$run", ptr @"main.(*Task).Run", ptr @main.Task.Run }] }, align 8 @0 = private constant [9 x i8] c"main.Task", align 1 diff --git a/ssa/abitype.go b/ssa/abitype.go index 7c6c1eebfa..6730823afc 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -458,6 +458,21 @@ func (b Builder) abiUncommonMethodSet(t types.Type) (mset *types.MethodSet, ok b return } +func abiUncommonMethodSetForDeclaration(t types.Type) (mset *types.MethodSet, ok bool) { + switch t := types.Unalias(t).(type) { + case *types.Named: + if _, isInterface := t.Underlying().(*types.Interface); isInterface { + return &types.MethodSet{}, true + } + return types.NewMethodSet(t), true + case *types.Struct, *types.Pointer: + if mset := types.NewMethodSet(t); mset.Len() != 0 { + return mset, true + } + } + return +} + /* type UncommonType struct { PkgPath_ string // import path; empty for built-in types like int, string @@ -673,6 +688,65 @@ func (b Builder) abiType(t types.Type) Expr { return ret } +// AbiTypes snapshots the runtime type descriptors materialized by this +// Program without retaining any LLVM-owned values. +func (p Program) AbiTypes() []AbiTypeInfo { + if len(p.abiSymbol) == 0 { + return nil + } + names := make([]string, 0, len(p.abiSymbol)) + for name := range p.abiSymbol { + names = append(names, name) + } + sort.Strings(names) + ret := make([]AbiTypeInfo, 0, len(names)) + for _, name := range names { + sym := p.abiSymbol[name] + ret = append(ret, AbiTypeInfo{Name: name, Raw: sym.Raw}) + } + return ret +} + +// RegisterAbiTypes declares runtime type descriptors owned by other Programs. +// It records only Go type identity and target-local LLVM types; descriptor +// definitions remain in the package modules that materialized them. +func (p Package) RegisterAbiTypes(infos []AbiTypeInfo) { + builder := &aBuilder{Prog: p.Prog, Pkg: p} + for _, info := range infos { + if info.Raw == nil { + continue + } + if _, ok := p.Prog.abiSymbol[info.Name]; ok { + continue + } + + t := info.Raw + mset, hasUncommon := abiUncommonMethodSetForDeclaration(t) + var methods []*types.Selection + if hasUncommon { + methods = builder.abiInterfaceMethods(mset) + } + rt := p.Prog.rtNamed(p.Prog.abi.RuntimeName(t)) + var typ types.Type = rt + if hasUncommon { + ut := p.Prog.rtNamed("uncommonType") + mt := p.Prog.rtNamed("Method") + typ = types.NewStruct([]*types.Var{ + types.NewVar(token.NoPos, nil, "T", rt), + types.NewVar(token.NoPos, nil, "U", ut), + types.NewVar(token.NoPos, nil, "M", types.NewArray(mt, int64(len(methods)))), + }, nil) + } + p.Prog.abiSymbol[info.Name] = &AbiSymbol{ + Name: info.Name, + PkgPath: p.Path(), + Raw: t, + Typ: p.Prog.Type(types.NewPointer(typ), InGo), + MSet: mset, + } + } +} + func (p Package) getAbiTypesFor(name string, filter func(sym *AbiSymbol) bool) Expr { prog := p.Prog var names []string diff --git a/ssa/package.go b/ssa/package.go index dacee7f3b8..67ae8a4976 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -251,6 +251,13 @@ type AbiSymbol struct { MSet *types.MethodSet } +// AbiTypeInfo is the LLVM-free identity needed to declare one runtime type +// descriptor in another Program. +type AbiTypeInfo struct { + Name string + Raw types.Type +} + // A Program presents a program. type Program = *aProgram diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 05f998fae0..6049c26620 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -2745,6 +2745,57 @@ func TestInitAbiTypesForSubset(t *testing.T) { } } +func TestRegisterAbiTypesAcrossPrograms(t *testing.T) { + runtimePkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + newProgram := func() Program { + prog := NewProgram(nil) + prog.sizes = types.SizesFor("gc", runtime.GOARCH) + prog.SetRuntime(runtimePkg) + return prog + } + + source := newProgram() + defer source.Dispose() + sourcePkg := source.NewPackage("source", "example.com/source") + sourceFn := sourcePkg.NewFunc("source", NoArgsNoRet, InC) + sourceBuilder := sourceFn.MakeBody(1) + sourceBuilder.abiType(types.NewSlice(types.Typ[types.Int])) + named := types.NewNamed( + types.NewTypeName(token.NoPos, types.NewPackage("example.com/source", "source"), "Named", nil), + types.NewStruct(nil, nil), + nil, + ) + sourceBuilder.abiType(named) + sourceBuilder.Return() + infos := source.AbiTypes() + if len(infos) == 0 { + t.Fatal("source Program produced no ABI type snapshot") + } + + target := newProgram() + defer target.Dispose() + targetPkg := target.NewPackage("target", "example.com/target") + targetPkg.RegisterAbiTypes(infos) + targetPkg.RegisterAbiTypes(append(infos, AbiTypeInfo{})) + for _, info := range infos { + if _, ok := target.abiSymbol[info.Name]; !ok { + t.Fatalf("target Program did not recreate ABI type %q", info.Name) + } + } + if targetPkg.InitAbiTypes("init$abitypes") == nil { + t.Fatal("recreated ABI types did not produce typelist initializer") + } + for _, info := range infos { + global := targetPkg.Module().NamedGlobal(info.Name) + if global.IsNil() || !global.IsDeclaration() { + t.Fatalf("target descriptor %q is not an external declaration", info.Name) + } + } +} + func TestInitAbiTypesForEmptySelection(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("bar", "foo/bar") diff --git a/xtool/env/env.go b/xtool/env/env.go index 469b7e97f2..1c14c07121 100644 --- a/xtool/env/env.go +++ b/xtool/env/env.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "regexp" "strings" @@ -32,7 +33,19 @@ var ( ) func ExpandEnvToArgs(s string) []string { - r, config := expandEnvWithCmd(s) + r, config := expandEnvWithCmd(s, "", nil) + return expandedArgs(r, config) +} + +// ExpandEnvToArgsWith expands variables and supported helper commands using +// the supplied request directory and environment. A non-nil environ prevents +// subprocesses and variable expansion from consulting process-global state. +func ExpandEnvToArgsWith(s, dir string, environ []string) []string { + r, config := expandEnvWithCmd(s, dir, environ) + return expandedArgs(r, config) +} + +func expandedArgs(r string, config bool) []string { if r == "" { return nil } @@ -43,11 +56,11 @@ func ExpandEnvToArgs(s string) []string { } func ExpandEnv(s string) string { - r, _ := expandEnvWithCmd(s) + r, _ := expandEnvWithCmd(s, "", nil) return r } -func expandEnvWithCmd(s string) (string, bool) { +func expandEnvWithCmd(s, dir string, environ []string) (string, bool) { var config bool expanded := reSubcmd.ReplaceAllStringFunc(s, func(m string) string { subcmd := strings.TrimSpace(m[2 : len(m)-1]) @@ -61,7 +74,16 @@ func expandEnvWithCmd(s string) (string, bool) { var out []byte var err error - out, err = exec.Command(cmd, args[1:]...).Output() + executable := cmd + if environ != nil { + executable = lookPathInEnvironment(cmd, dir, environ) + } + command := exec.Command(executable, args[1:]...) + command.Dir = dir + if environ != nil { + command.Env = append([]string(nil), environ...) + } + out, err = command.Output() if err != nil { // TODO(kindy): log in verbose mode @@ -70,7 +92,46 @@ func expandEnvWithCmd(s string) (string, bool) { return strings.Replace(strings.TrimSpace(string(out)), "\n", " ", -1) }) - return strings.TrimSpace(os.Expand(expanded, os.Getenv)), config + lookup := os.Getenv + if environ != nil { + lookup = func(key string) string { + prefix := key + "=" + for i := len(environ) - 1; i >= 0; i-- { + if strings.HasPrefix(environ[i], prefix) { + return strings.TrimPrefix(environ[i], prefix) + } + } + return "" + } + } + return strings.TrimSpace(os.Expand(expanded, lookup)), config +} + +func lookPathInEnvironment(name, dir string, environ []string) string { + if strings.ContainsRune(name, filepath.Separator) { + return name + } + path := "" + prefix := "PATH=" + for i := len(environ) - 1; i >= 0; i-- { + if strings.HasPrefix(environ[i], prefix) { + path = strings.TrimPrefix(environ[i], prefix) + break + } + } + for _, entry := range filepath.SplitList(path) { + if entry == "" { + entry = "." + } + if !filepath.IsAbs(entry) && dir != "" { + entry = filepath.Join(dir, entry) + } + candidate := filepath.Join(entry, name) + if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return candidate + } + } + return name } func parseSubcmd(s string) []string { diff --git a/xtool/env/env_test.go b/xtool/env/env_test.go new file mode 100644 index 0000000000..51c485733d --- /dev/null +++ b/xtool/env/env_test.go @@ -0,0 +1,75 @@ +package env + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +func TestExpandEnvToArgsWithUsesExplicitEnvironment(t *testing.T) { + t.Setenv("LLGO_ENV_TEST", "ambient") + got := ExpandEnvToArgsWith("$LLGO_ENV_TEST", "", []string{"LLGO_ENV_TEST=request"}) + if want := []string{"request"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ExpandEnvToArgsWith = %q, want %q", got, want) + } +} + +func TestExpandEnvUsesProcessEnvironment(t *testing.T) { + t.Setenv("LLGO_ENV_TEST", "ambient") + if got := ExpandEnv("$LLGO_ENV_TEST"); got != "ambient" { + t.Fatalf("ExpandEnv = %q, want %q", got, "ambient") + } + if got := ExpandEnvToArgs("$LLGO_ENV_TEST"); !reflect.DeepEqual(got, []string{"ambient"}) { + t.Fatalf("ExpandEnvToArgs = %q, want %q", got, []string{"ambient"}) + } + if got := ExpandEnvToArgs(""); got != nil { + t.Fatalf("ExpandEnvToArgs(empty) = %q, want nil", got) + } +} + +func TestExpandEnvToArgsWithConfiguresSubprocess(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixture is Unix-only") + } + dir := t.TempDir() + tool := filepath.Join(dir, "pkg-config") + script := "#!/bin/sh\nprintf '%s' \"-L$LLGO_ENV_TEST -I$PWD\"\n" + if err := os.WriteFile(tool, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + got := ExpandEnvToArgsWith( + "$(pkg-config --libs fixture)", + dir, + []string{"PATH=" + dir, "LLGO_ENV_TEST=request"}, + ) + resolvedDir, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + want := []string{"-Lrequest", "-I" + resolvedDir} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ExpandEnvToArgsWith = %q, want %q", got, want) + } +} + +func TestLookPathInEnvironmentBoundaries(t *testing.T) { + dir := t.TempDir() + tool := filepath.Join(dir, "fixture-tool") + if err := os.WriteFile(tool, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + if got := lookPathInEnvironment("fixture-tool", dir, []string{"PATH=" + string(os.PathListSeparator)}); got != tool { + t.Fatalf("lookPathInEnvironment with empty entry = %q, want %q", got, tool) + } + if got := lookPathInEnvironment(filepath.Join("bin", "tool"), dir, nil); got != filepath.Join("bin", "tool") { + t.Fatalf("lookPathInEnvironment with separator = %q", got) + } + if got := lookPathInEnvironment("missing-tool", dir, []string{"PATH=" + t.TempDir()}); got != "missing-tool" { + t.Fatalf("lookPathInEnvironment missing tool = %q", got) + } + if got := ExpandEnvToArgsWith("$LLGO_ENV_MISSING", dir, []string{"PATH=" + dir}); got != nil { + t.Fatalf("missing explicit environment variable = %q, want nil", got) + } +}