compiler/runtime: add //llgo:tls and //llgo:gls package variables

cl/_testgo/localitycodegen/in.go

@@ -0,0 +1,77 @@
+// LITTEST
+package main
+
+// CHECK-DAG: @"{{.*}}localitycodegen.scalar" = thread_local global i64 0
+// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_cache" = thread_local global i64 0
+// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$guard" = thread_local global i8 0
+// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure_cache" = thread_local global i64 0
+// CHECK-NOT: RegisterLocalRoot
+// CHECK-NOT: localitycodegen.pointer" = thread_local
+// CHECK-NOT: localitycodegen.initialized" = thread_local
+
+// CHECK-LABEL: define ptr @"{{.*}}localitycodegen.__llgo_local_block"()
+// CHECK: load i64, ptr @"{{.*}}localitycodegen.__llgo_local_cache"
+// CHECK: icmp ne i64
+// CHECK: ret ptr
+// CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_cache", i64 16, i64 8)
+// CHECK: ret ptr
+
+// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init"()
+// CHECK: call void @"{{.*}}localitycodegen.__llgo_local_init_0"()
+
+// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
+// CHECK: load i8, ptr
+// CHECK: call void @"{{.*}}runtime.EnsureLocalInitializer"(ptr @"{{.*}}localitycodegen.__llgo_tls_init$guard", ptr @"{{.*}}localitycodegen.__llgo_tls_init$failure_cache"
+
+// CHECK-LABEL: define ptr @{{"?ExportedLocality"?}}()
+// CHECK: call i64 @"{{.*}}EnterLocalContext"
+// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()
+// CHECK: call void @"{{.*}}LeaveLocalContext"
+// CHECK: ret ptr
+
+// CHECK-LABEL: define void @"{{.*}}localitycodegen.init"()
+// CHECK: store i8 2, ptr
+// CHECK: call ptr @"{{.*}}localitycodegen.newPointer"()
+// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
+// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()
+
+// CHECK-LABEL: define { i64, ptr, ptr } @"{{.*}}localitycodegen.values"()
+// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
+// CHECK: load i64, ptr @"{{.*}}localitycodegen.scalar"
+// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()
+// CHECK: load ptr, ptr
+// CHECK: load ptr, ptr
+
+// CHECK-LABEL: define ptr @"{{.*}}localitycodegen._llgo_routine$1"(ptr %0)
+// CHECK: alloca %"{{.*}}LocalContext", align 8
+// CHECK: call i64 @"{{.*}}EnterLocalContext"
+// CHECK: call void @"{{.*}}LeaveLocalContext"
+
+var backing int
+
+func newPointer() *int {
+	return &backing
+}
+
+//llgo:tls
+var scalar int
+
+//llgo:gls
+var pointer *int
+
+//llgo:tls
+var initialized = newPointer()
+
+func values() (int, *int, *int) {
+	return scalar, pointer, initialized
+}
+
+//export ExportedLocality
+func ExportedLocality() *int {
+	return pointer
+}
+
+func main() {
+	_, _, _ = values()
+	go values()
+}

cl/compile.go

@@ -210,6 +210,7 @@ type context struct {
 	staticGlobalInits map[*ssa.Global]llssa.Expr
 	staticInitStores  map[*ssa.Store]none
 	staticInitInstrs  map[ssa.Instruction]none
+	locality          localityLowering
 }
 
 func (p *context) rewriteValue(name string) (string, bool) {
@@ -391,7 +392,10 @@ func (p *context) compileGlobal(pkg llssa.Package, gbl *ssa.Global) {
 		return
 	}
 	dbgInstrln("==> NewVar", name, typ)
-	g := pkg.NewVar(name, typ, llssa.Background(vtype))
+	g, skip := p.localityGlobalStorage(pkg, gbl, name, typ, llssa.Background(vtype))
+	if skip {
+		return
+	}
 	if p.tryEmbedGlobalInit(pkg, gbl, g, name) {
 		return
 	}
@@ -601,12 +605,15 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 		dbgSymsEnabled := enableDbgSyms && (f == nil || f.Origin() == nil)
 		p.inits = append(p.inits, func() {
 			oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark := p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark
+			oldLocalityFunction := p.locality.function
 			p.fn = fn
 			p.goFn = f
 			p.callerFrameMark = llssa.Nil
+			p.locality.function = localityFunction{}
 			p.state = state // restore pkgState when compiling funcBody
 			defer func() {
 				p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark = oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark
+				p.locality.function = oldLocalityFunction
 			}()
 			p.phis = nil
 			if dbgSymsEnabled {
@@ -624,6 +631,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 				bodyPos := p.getFuncBodyPos(f)
 				b.DebugFunction(fn, debugFunctionScope(f), pos, bodyPos)
 			}
+			p.prepareExportedLocalContext(f)
 			p.bvals = make(map[ssa.Value]llssa.Expr)
 			p.methodNilDerefChecks = collectMethodNilDerefChecks(f)
 			off := make([]int, len(f.Blocks))
@@ -832,6 +840,9 @@ func (p *context) debugParams(b llssa.Builder, f *ssa.Function) {
 }
 
 func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, doModInit bool) llssa.BasicBlock {
+	oldLocalBlock := p.locality.function.block
+	p.locality.function.block = block
+	defer func() { p.locality.function.block = oldLocalBlock }()
 	var last int
 	var pyModInit bool
 	var prog = p.prog
@@ -840,6 +851,9 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do
 	var instrs = block.Instrs[n:]
 	var ret = fn.Block(block.Index)
 	b.SetBlock(ret)
+	if block.Index == 0 {
+		p.enterExportedLocalContext(b)
+	}
 	if block.Index == 0 && p.shouldTrackCallerFrames() {
 		p.pushCallerLocationFrame(b, block.Parent())
 	}
@@ -852,6 +866,7 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do
 	}
 
 	if doModInit {
+		p.initializeLocalGuards(b)
 		if p.state != pkgInPatch {
 			p.applyEmbedInits(b)
 		}
@@ -1706,6 +1721,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) {
 		if p.shouldTrackCallerFrames() {
 			p.popCallerLocationFrame(b)
 		}
+		p.leaveExportedLocalContext(b)
 		b.Return(results...)
 	case *ssa.If:
 		fn := p.fn
@@ -1804,7 +1820,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr {
 		if isCgoVar(varName) {
 			p.cgoSymbols = append(p.cgoSymbols, val.Name())
 		}
-		if enableDbgSyms {
+		if enableDbgSyms && p.localityAllowsGlobalDebug(v) {
 			pos := p.fset.Position(v.Pos())
 			b.DIGlobal(val, v.Name(), pos)
 		}
@@ -2068,6 +2084,15 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri
 		pkg.Pkg = pkgTypes
 		patch.Alt.Pkg = pkgTypes
 	}
+	if err = ParsePkgSyntax(prog, pkgProg.Fset, pkgTypes, files); err != nil {
+		return nil, nil, err
+	}
+	if err = prog.ValidateLocalities(llssa.PathOf(pkgTypes)); err != nil {
+		return nil, nil, err
+	}
+	if err = validateLocalInitializers(prog, pkgTypes); err != nil {
+		return nil, nil, err
+	}
 	if pkgPath == llssa.PkgRuntime {
 		prog.SetRuntime(pkgTypes)
 	}
@@ -2174,6 +2199,17 @@ func processPkg(ctx *context, ret llssa.Package, pkg *ssa.Package) {
 	sort.Slice(members, func(i, j int) bool {
 		return members[i].name < members[j].name
 	})
+	localGlobals := make([]*ssa.Global, 0)
+	for _, m := range members {
+		global, ok := m.val.(*ssa.Global)
+		if !ok || isCgoFuncPtrVar(global.Name()) {
+			continue
+		}
+		localGlobals = append(localGlobals, global)
+	}
+	// Address accessors and replay guards must exist before any function body
+	// can reference a local package variable, regardless of member sort order.
+	ctx.prepareLocalVariables(ret, localGlobals)
 
 	for _, m := range members {
 		member := m.val

cl/import.go

@@ -28,7 +28,9 @@ import (
 
 	"golang.org/x/tools/go/ssa"
 
+	"github.com/goplus/llgo/internal/directive"
 	"github.com/goplus/llgo/internal/env"
+	"github.com/goplus/llgo/internal/locality"
 	llssa "github.com/goplus/llgo/ssa"
 )
 
@@ -218,30 +220,6 @@ func (p *context) initFiles(pkgPath string, files []*ast.File, cPkg bool) {
 	}
 }
 
-// PreCollectLinknames scans syntax files before SSA compilation and populates
-// prog.Linkname for package-level //go:linkname / //llgo:link declarations.
-// It intentionally ignores //export because there is no package export context
-// during the pre-collection phase.
-func PreCollectLinknames(prog llssa.Program, pkgPath string, files []*ast.File) {
-	ctx := &context{prog: prog}
-	for _, file := range files {
-		for _, decl := range file.Decls {
-			switch decl := decl.(type) {
-			case *ast.FuncDecl:
-				fullName, inPkgName := astFuncName(pkgPath, decl)
-				ctx.processLinknameByDoc(decl.Doc, fullName, inPkgName, false, false)
-			case *ast.GenDecl:
-				if decl.Tok == token.VAR && len(decl.Specs) == 1 {
-					if names := decl.Specs[0].(*ast.ValueSpec).Names; len(names) == 1 {
-						inPkgName := names[0].Name
-						ctx.processLinknameByDoc(decl.Doc, pkgPath+"."+inPkgName, inPkgName, true, false)
-					}
-				}
-			}
-		}
-	}
-}
-
 // Collect skip names and skip other annotations, such as go: and llgo:
 // llgo:skip symbol1 symbol2 ...
 // llgo:skipall
@@ -298,6 +276,21 @@ func (p *context) collectSkip(line string, prefix int) {
 	}
 }
 
+func collectLinknameByDoc(prog llssa.Program, doc *ast.CommentGroup, fullName, inPkgName string) {
+	directives := directive.ParseGroup(doc)
+	for n := len(directives) - 1; n >= 0; n-- {
+		directive := directives[n]
+		if directive.Name != "go:linkname" && directive.Name != "llgo:link" {
+			continue
+		}
+		fields := strings.Fields(directive.Args)
+		if len(fields) >= 2 && fields[0] == inPkgName {
+			prog.SetLinkname(fullName, strings.Join(fields[1:], " "))
+			return
+		}
+	}
+}
+
 func (p *context) processLinknameByDoc(doc *ast.CommentGroup, fullName, inPkgName string, isVar, allowExport bool) bool {
 	if doc != nil {
 		for n := len(doc.List) - 1; n >= 0; n-- {
@@ -722,6 +715,9 @@ func (p *context) varOf(b llssa.Builder, v *ssa.Global) llssa.Expr {
 		}
 		panic("unreachable")
 	}
+	if local, ok := p.localVariableAddress(b, v, name); ok {
+		return local
+	}
 	ret := pkg.VarOf(name)
 	if ret == nil {
 		ret = pkg.NewVar(name, p.patchType(v.Type()), llssa.Background(vtype))
@@ -754,24 +750,59 @@ func (p *context) initPyModule() {
 	}
 }
 
-// ParsePkgSyntax parses AST of a package to check package-level compiler directives.
-func ParsePkgSyntax(prog llssa.Program, pkg *types.Package, files []*ast.File) {
+// ParsePkgSyntax collects declaration directives in one syntax pass before SSA
+// creation. Directives that need an LLVM package (such as //export) are applied
+// later by initFiles.
+func ParsePkgSyntax(prog llssa.Program, fset *token.FileSet, pkg *types.Package, files []*ast.File) error {
+	if pkg == nil {
+		return nil
+	}
+	if prog.PackageSyntaxParsed(pkg) {
+		return nil
+	}
 	ctx := &context{prog: prog}
 	pkgPath := llssa.PathOf(pkg)
 	for _, file := range files {
 		for _, decl := range file.Decls {
 			switch decl := decl.(type) {
 			case *ast.FuncDecl:
-				fullName, _ := astFuncName(pkgPath, decl)
+				if err := locality.ValidateDoc(fset, decl.Doc); err != nil {
+					return err
+				}
+				if err := locality.ValidateFuncBody(fset, decl.Body); err != nil {
+					return err
+				}
+				fullName, inPkgName := astFuncName(pkgPath, decl)
+				collectLinknameByDoc(prog, decl.Doc, fullName, inPkgName)
 				ctx.processNoInterfaceByDoc(decl.Doc, fullName)
 			case *ast.GenDecl:
-				switch decl.Tok {
-				case token.TYPE:
+				if decl.Tok == token.VAR {
+					if len(decl.Specs) == 1 {
+						if names := decl.Specs[0].(*ast.ValueSpec).Names; len(names) == 1 {
+							inPkgName := names[0].Name
+							collectLinknameByDoc(prog, decl.Doc, pkgPath+"."+inPkgName, inPkgName)
+						}
+					}
+					vars, err := locality.ScanPackageVar(fset, decl)
+					if err != nil {
+						return err
+					}
+					for _, variable := range vars {
+						prog.SetLocalityInfo(llssa.FullName(pkg, variable.Name), variable.Info)
+					}
+					continue
+				}
+				if err := locality.ValidateNonPackageVar(fset, decl); err != nil {
+					return err
+				}
+				if decl.Tok == token.TYPE {
 					handleTypeDecl(prog, pkg, decl)
 				}
 			}
 		}
 	}
+	prog.MarkPackageSyntaxParsed(pkg)
+	return nil
 }
 
 func handleTypeDecl(prog llssa.Program, pkg *types.Package, decl *ast.GenDecl) {