From 26ac3a716331668ba96e5178fefd73c91e7b9678 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 14 Jul 2026 06:44:29 +0800 Subject: [PATCH 01/13] compiler: add standalone locality directive analysis --- internal/directive/directive.go | 77 +++++ internal/directive/directive_test.go | 47 +++ internal/locality/locality.go | 90 +++++ internal/locality/locality_test.go | 479 +++++++++++++++++++++++++++ internal/locality/prepare.go | 221 ++++++++++++ internal/locality/scan.go | 193 +++++++++++ 6 files changed, 1107 insertions(+) create mode 100644 internal/directive/directive.go create mode 100644 internal/directive/directive_test.go create mode 100644 internal/locality/locality.go create mode 100644 internal/locality/locality_test.go create mode 100644 internal/locality/prepare.go create mode 100644 internal/locality/scan.go diff --git a/internal/directive/directive.go b/internal/directive/directive.go new file mode 100644 index 0000000000..2678eb69ff --- /dev/null +++ b/internal/directive/directive.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package directive parses Go and LLGo source directives without assigning +// feature-specific semantics to them. +package directive + +import ( + "go/ast" + "go/token" + "strings" +) + +// Directive is one normalized Go or LLGo source directive. +type Directive struct { + Name string + Args string + Raw string + Pos token.Pos +} + +// Parse normalizes comment when it uses a supported directive spelling. +func Parse(comment *ast.Comment) (Directive, bool) { + if comment == nil { + return Directive{}, false + } + raw := comment.Text + var namespace, body string + switch { + case strings.HasPrefix(raw, "//go:"): + namespace, body = "go:", raw[len("//go:"):] + case strings.HasPrefix(raw, "//llgo:"): + namespace, body = "llgo:", raw[len("//llgo:"):] + case strings.HasPrefix(raw, "// llgo:"): + namespace, body = "llgo:", raw[len("// llgo:"):] + case strings.HasPrefix(raw, "//export "): + return Directive{Name: "export", Args: strings.TrimSpace(raw[len("//export "):]), Raw: raw, Pos: comment.Pos()}, true + default: + return Directive{}, false + } + body = strings.TrimSpace(body) + if body == "" { + return Directive{}, false + } + name, args := body, "" + if idx := strings.IndexAny(body, " \t"); idx >= 0 { + name, args = body[:idx], strings.TrimSpace(body[idx+1:]) + } + return Directive{Name: namespace + name, Args: args, Raw: raw, Pos: comment.Pos()}, true +} + +// ParseGroup returns all normalized directives in doc in source order. +func ParseGroup(doc *ast.CommentGroup) []Directive { + if doc == nil { + return nil + } + ret := make([]Directive, 0, len(doc.List)) + for _, comment := range doc.List { + if parsed, ok := Parse(comment); ok { + ret = append(ret, parsed) + } + } + return ret +} diff --git a/internal/directive/directive_test.go b/internal/directive/directive_test.go new file mode 100644 index 0000000000..c9bb47fd07 --- /dev/null +++ b/internal/directive/directive_test.go @@ -0,0 +1,47 @@ +package directive + +import ( + "go/ast" + "testing" +) + +func TestParse(t *testing.T) { + tests := []struct { + text string + name string + args string + ok bool + }{ + {text: "// ordinary"}, + {text: "//go:"}, + {text: "//go:noinline", name: "go:noinline", ok: true}, + {text: "//llgo:tls", name: "llgo:tls", ok: true}, + {text: "// llgo:type C", name: "llgo:type", args: "C", ok: true}, + {text: "//llgo:link\tF C.f", name: "llgo:link", args: "F C.f", ok: true}, + {text: "//export F", name: "export", args: "F", ok: true}, + } + if _, ok := Parse(nil); ok { + t.Fatal("nil comment parsed as a directive") + } + if ParseGroup(nil) != nil { + t.Fatal("nil comment group returned directives") + } + for _, test := range tests { + got, ok := Parse(&ast.Comment{Text: test.text}) + if ok != test.ok || got.Name != test.name || got.Args != test.args || got.Raw != map[bool]string{true: test.text}[test.ok] { + t.Fatalf("Parse(%q) = %+v, %v", test.text, got, ok) + } + } +} + +func TestParseGroupPreservesSourceOrder(t *testing.T) { + doc := &ast.CommentGroup{List: []*ast.Comment{ + {Text: "// ordinary"}, + {Text: "//go:noinline"}, + {Text: "//llgo:tls"}, + }} + got := ParseGroup(doc) + if len(got) != 2 || got[0].Name != "go:noinline" || got[1].Name != "llgo:tls" { + t.Fatalf("ParseGroup = %+v", got) + } +} diff --git a/internal/locality/locality.go b/internal/locality/locality.go new file mode 100644 index 0000000000..1e9a7bfca3 --- /dev/null +++ b/internal/locality/locality.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package locality defines the source-level TLS and GLS declaration model. +// It intentionally has no dependency on LLGo's SSA or LLVM lowering layers. +package locality + +import "fmt" + +const ( + ThreadDirective = "//llgo:tls" + GoroutineDirective = "//llgo:gls" + InitPrefix = "__llgo_local_init_" +) + +// Kind identifies the execution context that owns a package variable. +type Kind uint8 + +const ( + None Kind = iota + Thread + Goroutine +) + +// Info is the locality-specific part of a declaration's compiler metadata. +type Info struct { + Locality Kind + HasInitializer bool + InitFunc string + InitOrder int +} + +func (kind Kind) String() string { + switch kind { + case None: + return "" + case Thread: + return "tls" + case Goroutine: + return "gls" + default: + return fmt.Sprintf("invalid:%d", kind) + } +} + +// Parse converts the cache representation of a locality into a Kind. +func Parse(name string) (Kind, bool) { + switch name { + case "": + return None, true + case "tls": + return Thread, true + case "gls": + return Goroutine, true + default: + return None, false + } +} + +// Directive returns the source directive for kind. +func Directive(kind Kind) string { + if kind == Goroutine { + return GoroutineDirective + } + return ThreadDirective +} + +// Merge combines declaration- and spec-level locality directives. +func Merge(a, b Kind) (Kind, bool) { + if a != None && b != None && a != b { + return None, false + } + if b != None { + return b, true + } + return a, true +} diff --git a/internal/locality/locality_test.go b/internal/locality/locality_test.go new file mode 100644 index 0000000000..4a8ac7d44b --- /dev/null +++ b/internal/locality/locality_test.go @@ -0,0 +1,479 @@ +package locality + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +func TestKindEncodingAndNames(t *testing.T) { + tests := []struct { + kind Kind + name string + }{ + {None, ""}, + {Thread, "tls"}, + {Goroutine, "gls"}, + } + for _, test := range tests { + if got := test.kind.String(); got != test.name { + t.Fatalf("Kind(%d).String() = %q, want %q", test.kind, got, test.name) + } + if got, ok := Parse(test.name); !ok || got != test.kind { + t.Fatalf("Parse(%q) = %v, %v", test.name, got, ok) + } + } + if got := Kind(99).String(); got != "invalid:99" { + t.Fatalf("invalid locality name = %q", got) + } + if _, ok := Parse("invalid:99"); ok { + t.Fatal("Parse accepted invalid locality") + } + if Directive(Thread) != ThreadDirective || Directive(Goroutine) != GoroutineDirective { + t.Fatal("unexpected locality directive names") + } +} + +func TestMerge(t *testing.T) { + tests := []struct { + a, b Kind + want Kind + ok bool + }{ + {want: None, ok: true}, + {a: Thread, want: Thread, ok: true}, + {b: Goroutine, want: Goroutine, ok: true}, + {a: Thread, b: Thread, want: Thread, ok: true}, + {a: Thread, b: Goroutine, want: None}, + } + for _, test := range tests { + got, ok := Merge(test.a, test.b) + if got != test.want || ok != test.ok { + t.Fatalf("Merge(%v, %v) = %v, %v", test.a, test.b, got, ok) + } + } +} + +func TestScanPackageVar(t *testing.T) { + fset, file := parseFile(t, `package p + +//llgo:tls +var ( + first int + //llgo:gls + second int +) +`) + decl := file.Decls[0].(*ast.GenDecl) + if _, err := ScanPackageVar(fset, decl); err == nil || !strings.Contains(err.Error(), "cannot apply to the same variable declaration") { + t.Fatalf("ScanPackageVar conflict error = %v", err) + } + + fset, file = parseFile(t, `package p + +var ( + //llgo:tls + first, second = 1, 2 +) +`) + vars, err := ScanPackageVar(fset, file.Decls[0].(*ast.GenDecl)) + if err != nil { + t.Fatal(err) + } + if len(vars) != 2 || vars[0].Info.Locality != Thread || !vars[0].Info.HasInitializer || vars[1].Name != "second" { + t.Fatalf("ScanPackageVar = %+v", vars) + } +} + +func TestScanPackageVarBranches(t *testing.T) { + comment := func(text string) *ast.CommentGroup { + doc := &ast.CommentGroup{} + for _, line := range strings.Split(text, "\n") { + doc.List = append(doc.List, &ast.Comment{Text: line}) + } + return doc + } + tests := []struct { + name string + decl *ast.GenDecl + want string + }{ + { + name: "declaration error", + decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//llgo:tls extra")}, + want: "does not accept arguments", + }, + { + name: "spec error", + decl: &ast.GenDecl{Tok: token.VAR, Specs: []ast.Spec{&ast.ValueSpec{Doc: comment("//llgo:gls extra")}}}, + want: "does not accept arguments", + }, + { + name: "embed conflict", + decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//llgo:tls\n//go:embed value.txt"), Specs: []ast.Spec{&ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("Value")}}}}, + want: "//go:embed", + }, + { + name: "blank name", + decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//llgo:tls"), Specs: []ast.Spec{&ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("_")}}}}, + want: "blank identifier", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := ScanPackageVar(nil, test.decl); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ScanPackageVar error = %v, want %q", err, test.want) + } + }) + } + + ordinary := &ast.GenDecl{Tok: token.VAR, Specs: []ast.Spec{ + &ast.ImportSpec{}, + &ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("Value")}}, + }} + if vars, err := ScanPackageVar(nil, ordinary); err != nil || len(vars) != 0 { + t.Fatalf("ordinary ScanPackageVar = %+v, %v", vars, err) + } +} + +func TestDirectivePlacementDiagnostics(t *testing.T) { + tests := []struct { + name string + src string + want string + }{ + { + name: "grouped import spec", + src: `package p +import ( + //llgo:tls + "unsafe" +) +var _ = unsafe.Sizeof(0) +`, + want: "package-level var", + }, + { + name: "local var", + src: `package p +func f() { + //llgo:gls + var value int + _ = value +} +`, + want: "package-level var", + }, + { + name: "nested function local", + src: `package p +func f() { + _ = func() { + //llgo:tls extra + var value int + _ = value + } +} +`, + want: "does not accept arguments", + }, + { + name: "nested function in local initializer", + src: `package p +func f() { + var nested = func() { + //llgo:gls + var value int + _ = value + } + _ = nested +} +`, + want: "package-level var", + }, + { + name: "function", + src: `package p +//llgo:tls +func f() {} +`, + want: "package-level var", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fset, file := parseFile(t, test.src) + err := validateFile(fset, file) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("validation error = %v, want %q", err, test.want) + } + }) + } +} + +func TestDirectiveDiagnostics(t *testing.T) { + tests := []struct { + comment string + want string + }{ + {"//llgo:threadlocal", "use //llgo:tls"}, + {"//llgo:goroutinelocal", "use //llgo:gls"}, + {"//llgo:tls extra", "does not accept arguments"}, + {"//llgo:tls\n//llgo:gls", "cannot apply to the same variable declaration"}, + } + for _, test := range tests { + doc := &ast.CommentGroup{} + for _, line := range strings.Split(test.comment, "\n") { + doc.List = append(doc.List, &ast.Comment{Text: line}) + } + if _, _, err := FromDoc(nil, doc); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("FromDoc(%q) error = %v", test.comment, err) + } + } + if err := ValidateDoc(nil, nil); err != nil { + t.Fatal(err) + } + if kind, _, err := FromDoc(nil, &ast.CommentGroup{List: []*ast.Comment{{Text: "//go:noinline"}}}); err != nil || kind != None { + t.Fatalf("ordinary directive = %v, %v", kind, err) + } + if err := ValidateFuncBody(nil, nil); err != nil { + t.Fatal(err) + } + typeDecl := &ast.GenDecl{Tok: token.TYPE, Specs: []ast.Spec{ + &ast.TypeSpec{Name: ast.NewIdent("T"), Doc: &ast.CommentGroup{List: []*ast.Comment{{Text: "//llgo:tls"}}}}, + }} + if err := ValidateNonPackageVar(nil, typeDecl); err == nil || !strings.Contains(err.Error(), "package-level var") { + t.Fatalf("type spec validation error = %v", err) + } + if hasDirective(&ast.CommentGroup{List: []*ast.Comment{{Text: "//go:noinline"}}}, "go:embed") { + t.Fatal("hasDirective matched an unrelated directive") + } +} + +func TestInitializerLocalityDiagnostics(t *testing.T) { + pkg := types.NewPackage("example.com/p", "p") + thread := types.NewVar(token.NoPos, pkg, "thread", types.Typ[types.Int]) + goroutine := types.NewVar(token.NoPos, pkg, "goroutine", types.Typ[types.Int]) + ordinary := types.NewVar(token.NoPos, pkg, "ordinary", types.Typ[types.Int]) + vars := map[string]Info{ + "thread": {Locality: Thread, HasInitializer: true}, + "goroutine": {Locality: Goroutine, HasInitializer: true}, + } + rhs := ast.NewIdent("rhs") + tests := []struct { + lhs []*types.Var + want string + }{ + {[]*types.Var{thread, ordinary}, "mix local and ordinary"}, + {[]*types.Var{ordinary, thread}, "mix local and ordinary"}, + {[]*types.Var{thread, goroutine}, "mix thread-local and goroutine-local"}, + } + for _, test := range tests { + if _, _, err := initializerLocality(nil, &types.Initializer{Lhs: test.lhs, Rhs: rhs}, vars); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("initializerLocality error = %v, want %q", err, test.want) + } + } + if kind, found, err := initializerLocality(nil, &types.Initializer{Lhs: []*types.Var{ordinary}, Rhs: rhs}, vars); err != nil || found || kind != None { + t.Fatalf("ordinary initializer = %v, %v, %v", kind, found, err) + } +} + +func TestPrepareIsIdempotentAcrossPrograms(t *testing.T) { + fset, file := parseFile(t, `package p +func makeValue() *int { value := 42; return &value } +//llgo:tls +var Value = makeValue() +`) + files := []*ast.File{file} + info := newTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/p", fset, files, info) + if err != nil { + t.Fatal(err) + } + vars, err := ScanPackageVar(fset, file.Decls[1].(*ast.GenDecl)) + if err != nil { + t.Fatal(err) + } + raw := make(map[string]Info) + for _, variable := range vars { + raw[variable.Name] = variable.Info + } + + prepared, err := Prepare(fset, pkg.Path(), pkg, info, files, raw) + if err != nil { + t.Fatal(err) + } + declCount := len(file.Decls) + scopeCount := len(pkg.Scope().Names()) + initName := prepared["Value"].InitFunc + if initName != "example.com/p.__llgo_local_init_0" || prepared["Value"].InitOrder != 1 { + t.Fatalf("prepared metadata = %+v", prepared["Value"]) + } + + again, err := Prepare(fset, pkg.Path(), pkg, info, files, prepared) + if err != nil { + t.Fatal(err) + } + reused, err := Prepare(fset, pkg.Path(), pkg, info, files, raw) + if err != nil { + t.Fatal(err) + } + if len(file.Decls) != declCount || len(pkg.Scope().Names()) != scopeCount { + t.Fatalf("repeated Prepare changed syntax/scope: decls=%d/%d scope=%d/%d", len(file.Decls), declCount, len(pkg.Scope().Names()), scopeCount) + } + if again["Value"] != prepared["Value"] || reused["Value"] != prepared["Value"] { + t.Fatalf("repeated metadata = %+v, reused = %+v, want %+v", again["Value"], reused["Value"], prepared["Value"]) + } +} + +func TestPrepareZeroValuePointerAndValidation(t *testing.T) { + pkg := types.NewPackage("example.com/p", "p") + value := types.NewVar(token.NoPos, pkg, "Value", types.NewPointer(types.Typ[types.Int])) + pkg.Scope().Insert(value) + vars := map[string]Info{"Value": {Locality: Goroutine}} + prepared, err := Prepare(nil, pkg.Path(), pkg, &types.Info{}, nil, vars) + if err != nil { + t.Fatal(err) + } + if got := prepared["Value"]; got.InitFunc != "" || got.InitOrder != 0 { + t.Fatalf("zero-value initializer = %+v", got) + } + if err := ValidatePrepared(pkg.Path(), map[string]Info{"Value": {Locality: Thread, HasInitializer: true}}); err == nil { + t.Fatal("ValidatePrepared accepted missing initializer metadata") + } + if err := ValidatePrepared(pkg.Path(), map[string]Info{"Value": {Locality: Thread, InitFunc: "p.init", InitOrder: 1}}); err == nil { + t.Fatal("ValidatePrepared accepted initializer metadata on a zero-value declaration") + } + if err := ValidatePrepared(pkg.Path(), map[string]Info{ + "Ordinary": {}, + "Value": {Locality: Thread, HasInitializer: true, InitFunc: "p.init", InitOrder: 1}, + }); err != nil { + t.Fatal(err) + } +} + +func TestPrepareEarlyReturnsAndMissingFiles(t *testing.T) { + if got, err := Prepare(nil, "", nil, nil, nil, nil); err != nil || len(got) != 0 { + t.Fatalf("nil Prepare = %+v, %v", got, err) + } + pkg := types.NewPackage("example.com/p", "p") + value := types.NewVar(token.NoPos, pkg, "Value", types.Typ[types.Int]) + pkg.Scope().Insert(value) + info := &types.Info{InitOrder: []*types.Initializer{{Lhs: []*types.Var{value}, Rhs: ast.NewIdent("rhs")}}} + vars := map[string]Info{"Value": {Locality: Thread, HasInitializer: true}} + if _, err := Prepare(nil, pkg.Path(), pkg, info, nil, vars); err == nil || !strings.Contains(err.Error(), "without syntax files") { + t.Fatalf("Prepare without files error = %v", err) + } + if got, err := Prepare(nil, pkg.Path(), pkg, &types.Info{}, nil, nil); err != nil || len(got) != 0 { + t.Fatalf("Prepare without localities = %+v, %v", got, err) + } +} + +func TestPrepareInitializerBranches(t *testing.T) { + pkg := types.NewPackage("example.com/p", "p") + local := types.NewVar(token.NoPos, pkg, "Local", types.Typ[types.Int]) + ordinary := types.NewVar(token.NoPos, pkg, "Ordinary", types.Typ[types.Int]) + pkg.Scope().Insert(local) + pkg.Scope().Insert(ordinary) + rhs := ast.NewIdent("rhs") + info := &types.Info{InitOrder: []*types.Initializer{ + {Lhs: []*types.Var{ordinary}, Rhs: ast.NewIdent("ordinary")}, + {Lhs: []*types.Var{local, ordinary}, Rhs: rhs}, + }} + vars := map[string]Info{"Local": {Locality: Thread, HasInitializer: true}} + if _, err := Prepare(nil, pkg.Path(), pkg, info, []*ast.File{{}}, vars); err == nil || !strings.Contains(err.Error(), "mix local and ordinary") { + t.Fatalf("mixed Prepare error = %v", err) + } + + info.InitOrder = []*types.Initializer{{Lhs: []*types.Var{local}, Rhs: rhs}} + prepared, err := Prepare(nil, pkg.Path(), pkg, info, []*ast.File{{}}, vars) + if err != nil { + t.Fatal(err) + } + if prepared["Local"].InitFunc == "" || info.Uses == nil || info.Defs == nil { + t.Fatalf("Prepare did not initialize type maps: %+v", prepared["Local"]) + } + + conflicting := &types.Initializer{Lhs: []*types.Var{local, ordinary}, Rhs: rhs} + if got := preparedInitName(conflicting, map[string]Info{ + "Local": {InitFunc: "p.first", InitOrder: 1}, + "Ordinary": {InitFunc: "p.second", InitOrder: 1}, + }, 1); got != "" { + t.Fatalf("preparedInitName accepted conflicting helpers: %q", got) + } + if got := qualify("", "Value"); got != "Value" { + t.Fatalf("qualify empty package = %q", got) + } +} + +func TestFindLocalInitializerRejectsLookalikes(t *testing.T) { + pkg := types.NewPackage("example.com/p", "p") + value := types.NewVar(token.NoPos, pkg, "Value", types.Typ[types.Int]) + rhs := ast.NewIdent("rhs") + initializer := &types.Initializer{Lhs: []*types.Var{value}, Rhs: rhs} + files := []*ast.File{{Decls: []ast.Decl{ + &ast.FuncDecl{ + Name: ast.NewIdent(InitPrefix + "0"), + Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: rhs}}}, + }, + &ast.FuncDecl{ + Name: ast.NewIdent(InitPrefix + "1"), + Body: &ast.BlockStmt{List: []ast.Stmt{&ast.AssignStmt{ + Lhs: []ast.Expr{&ast.BasicLit{Kind: token.INT, Value: "1"}}, + Tok: token.ASSIGN, + Rhs: []ast.Expr{rhs}, + }}}, + }, + }}} + if got := findLocalInitializer(pkg, &types.Info{}, files, initializer); got != "" { + t.Fatalf("findLocalInitializer accepted lookalike %q", got) + } +} + +func parseFile(t *testing.T, src string) (*token.FileSet, *ast.File) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "source.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + return fset, file +} + +func validateFile(fset *token.FileSet, file *ast.File) error { + for _, node := range file.Decls { + switch decl := node.(type) { + case *ast.FuncDecl: + if err := ValidateDoc(fset, decl.Doc); err != nil { + return err + } + if err := ValidateFuncBody(fset, decl.Body); err != nil { + return err + } + case *ast.GenDecl: + if decl.Tok == token.VAR { + if _, err := ScanPackageVar(fset, decl); err != nil { + return err + } + } else if err := ValidateNonPackageVar(fset, decl); err != nil { + return err + } + } + } + return nil +} + +func newTypeInfo() *types.Info { + return &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + Instances: make(map[*ast.Ident]types.Instance), + } +} diff --git a/internal/locality/prepare.go b/internal/locality/prepare.go new file mode 100644 index 0000000000..429dd78834 --- /dev/null +++ b/internal/locality/prepare.go @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package locality + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "sort" +) + +// Prepare rewrites local package initializers into replayable synthetic +// functions and returns updated metadata. It is idempotent for repeated calls, +// including calls made by another compiler Program reusing the same syntax and +// types.Package objects. +func Prepare(fset *token.FileSet, pkgPath string, pkg *types.Package, typeInfo *types.Info, files []*ast.File, vars map[string]Info) (map[string]Info, error) { + ret := cloneInfo(vars) + if pkg == nil || typeInfo == nil || !hasLocality(ret) { + return ret, nil + } + + nextName := 0 + for order, initializer := range typeInfo.InitOrder { + _, found, err := initializerLocality(fset, initializer, ret) + if err != nil { + return nil, err + } + if !found { + continue + } + initOrder := order + 1 + if initName := preparedInitName(initializer, ret, initOrder); initName != "" { + setInitializerNames(initializer, ret, initName, initOrder) + continue + } + if len(files) == 0 { + return nil, fmt.Errorf("cannot prepare local initializer for package %q without syntax files", pkgPath) + } + name := findLocalInitializer(pkg, typeInfo, files, initializer) + if name == "" { + for { + name = fmt.Sprintf("%s%d", InitPrefix, nextName) + nextName++ + if pkg.Scope().Lookup(name) == nil { + break + } + } + fnObj, decl := makeLocalInitializer(pkg, typeInfo, name, initializer) + pkg.Scope().Insert(fnObj) + files[len(files)-1].Decls = append(files[len(files)-1].Decls, decl) + } + setInitializerNames(initializer, ret, qualify(pkgPath, name), initOrder) + } + + return ret, nil +} + +// ValidatePrepared verifies that every explicit local initializer was prepared +// before Go SSA construction. +func ValidatePrepared(pkgPath string, vars map[string]Info) error { + names := make([]string, 0, len(vars)) + for name := range vars { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + info := vars[name] + if info.Locality == None { + continue + } + prepared := info.InitFunc != "" && info.InitOrder != 0 + if info.HasInitializer != prepared { + return fmt.Errorf("local variable %s has inconsistent initializer metadata before SSA compilation", qualify(pkgPath, name)) + } + } + return nil +} + +func initializerLocality(fset *token.FileSet, initializer *types.Initializer, vars map[string]Info) (Kind, bool, error) { + var kind Kind + localCount := 0 + for _, variable := range initializer.Lhs { + info, ok := vars[variable.Name()] + if !ok || info.Locality == None { + if localCount != 0 { + return None, false, errorAt(fset, initializer.Rhs.Pos(), "one initializer cannot mix local and ordinary package variables") + } + continue + } + if localCount == 0 { + kind = info.Locality + } else if kind != info.Locality { + return None, false, errorAt(fset, initializer.Rhs.Pos(), "one initializer cannot mix thread-local and goroutine-local variables") + } + localCount++ + } + if localCount != 0 && len(initializer.Lhs) != localCount { + return None, false, errorAt(fset, initializer.Rhs.Pos(), "one initializer cannot mix local and ordinary package variables") + } + return kind, localCount != 0, nil +} + +func preparedInitName(initializer *types.Initializer, vars map[string]Info, order int) string { + var name string + for _, variable := range initializer.Lhs { + info := vars[variable.Name()] + if info.InitFunc == "" || info.InitOrder != order { + return "" + } + if name == "" { + name = info.InitFunc + } else if name != info.InitFunc { + return "" + } + } + return name +} + +func findLocalInitializer(pkg *types.Package, info *types.Info, files []*ast.File, initializer *types.Initializer) string { + for _, file := range files { + for _, node := range file.Decls { + decl, ok := node.(*ast.FuncDecl) + if !ok || len(decl.Name.Name) < len(InitPrefix) || decl.Name.Name[:len(InitPrefix)] != InitPrefix || decl.Body == nil || len(decl.Body.List) != 1 { + continue + } + assign, ok := decl.Body.List[0].(*ast.AssignStmt) + if !ok || assign.Tok != token.ASSIGN || len(assign.Rhs) != 1 || assign.Rhs[0] != initializer.Rhs || len(assign.Lhs) != len(initializer.Lhs) { + continue + } + matches := true + for i, lhs := range assign.Lhs { + ident, ok := lhs.(*ast.Ident) + if !ok || info.Uses[ident] != initializer.Lhs[i] { + matches = false + break + } + } + if matches { + if object := pkg.Scope().Lookup(decl.Name.Name); object == info.Defs[decl.Name] { + return decl.Name.Name + } + } + } + } + return "" +} + +func makeLocalInitializer(pkg *types.Package, info *types.Info, name string, initializer *types.Initializer) (*types.Func, *ast.FuncDecl) { + if info.Uses == nil { + info.Uses = make(map[*ast.Ident]types.Object) + } + if info.Defs == nil { + info.Defs = make(map[*ast.Ident]types.Object) + } + lhs := make([]ast.Expr, len(initializer.Lhs)) + for i, variable := range initializer.Lhs { + ident := ast.NewIdent(variable.Name()) + info.Uses[ident] = variable + lhs[i] = ident + } + nameIdent := ast.NewIdent(name) + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + fnObj := types.NewFunc(token.NoPos, pkg, name, sig) + info.Defs[nameIdent] = fnObj + decl := &ast.FuncDecl{ + Name: nameIdent, + Type: &ast.FuncType{Params: &ast.FieldList{}}, + Body: &ast.BlockStmt{List: []ast.Stmt{ + &ast.AssignStmt{Lhs: lhs, Tok: token.ASSIGN, Rhs: []ast.Expr{initializer.Rhs}}, + }}, + } + return fnObj, decl +} + +func setInitializerNames(initializer *types.Initializer, vars map[string]Info, initName string, order int) { + for _, variable := range initializer.Lhs { + info := vars[variable.Name()] + info.InitFunc = initName + info.InitOrder = order + vars[variable.Name()] = info + } +} + +func cloneInfo(vars map[string]Info) map[string]Info { + ret := make(map[string]Info, len(vars)) + for name, info := range vars { + ret[name] = info + } + return ret +} + +func hasLocality(vars map[string]Info) bool { + for _, info := range vars { + if info.Locality != None { + return true + } + } + return false +} + +func qualify(pkgPath, name string) string { + if pkgPath == "" { + return name + } + return pkgPath + "." + name +} diff --git a/internal/locality/scan.go b/internal/locality/scan.go new file mode 100644 index 0000000000..b458a81795 --- /dev/null +++ b/internal/locality/scan.go @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package locality + +import ( + "fmt" + "go/ast" + "go/token" + + "github.com/goplus/llgo/internal/directive" +) + +const ( + legacyThread = "llgo:threadlocal" + legacyGoroutine = "llgo:goroutinelocal" +) + +// Variable records locality metadata collected for one package variable. +type Variable struct { + Name string + Info Info +} + +// ScanPackageVar validates and collects locality directives on a package-level +// var declaration. +func ScanPackageVar(fset *token.FileSet, decl *ast.GenDecl) ([]Variable, error) { + declKind, declPos, err := FromDoc(fset, decl.Doc) + if err != nil { + return nil, err + } + var ret []Variable + for _, node := range decl.Specs { + spec, ok := node.(*ast.ValueSpec) + if !ok { + continue + } + specKind, specPos, err := FromDoc(fset, spec.Doc) + if err != nil { + return nil, err + } + kind, _, err := mergeAt(fset, declKind, declPos, specKind, specPos) + if err != nil { + return nil, err + } + if kind == None { + continue + } + if hasDirective(decl.Doc, "go:embed") || hasDirective(spec.Doc, "go:embed") { + return nil, errorAt(fset, spec.Pos(), "%s and //go:embed cannot apply to the same variable declaration", Directive(kind)) + } + for _, ident := range spec.Names { + if ident.Name == "_" { + return nil, errorAt(fset, ident.Pos(), "locality directive cannot apply to the blank identifier") + } + ret = append(ret, Variable{ + Name: ident.Name, + Info: Info{Locality: kind, HasInitializer: len(spec.Values) != 0}, + }) + } + } + return ret, nil +} + +// ValidateNonPackageVar rejects locality directives on declarations other +// than package-level vars, including grouped import/type/const specs. +func ValidateNonPackageVar(fset *token.FileSet, decl *ast.GenDecl) error { + if err := ValidateDoc(fset, decl.Doc); err != nil { + return err + } + for _, node := range decl.Specs { + var doc *ast.CommentGroup + switch spec := node.(type) { + case *ast.ImportSpec: + doc = spec.Doc + case *ast.TypeSpec: + doc = spec.Doc + case *ast.ValueSpec: + doc = spec.Doc + } + if err := ValidateDoc(fset, doc); err != nil { + return err + } + } + return nil +} + +// ValidateFuncBody rejects locality directives on declarations nested inside +// a function or function literal. +func ValidateFuncBody(fset *token.FileSet, body *ast.BlockStmt) error { + if body == nil { + return nil + } + var firstErr error + ast.Inspect(body, func(node ast.Node) bool { + if firstErr != nil { + return false + } + stmt, ok := node.(*ast.DeclStmt) + if !ok { + return true + } + decl, ok := stmt.Decl.(*ast.GenDecl) + if ok { + firstErr = ValidateNonPackageVar(fset, decl) + } + return firstErr == nil + }) + return firstErr +} + +// FromDoc returns the locality directive attached to doc. +func FromDoc(fset *token.FileSet, doc *ast.CommentGroup) (Kind, token.Pos, error) { + var kind Kind + var pos token.Pos + for _, directive := range directive.ParseGroup(doc) { + var next Kind + switch directive.Name { + case "llgo:tls": + next = Thread + case "llgo:gls": + next = Goroutine + case legacyThread: + return None, token.NoPos, errorAt(fset, directive.Pos, "//%s is not supported; use %s", legacyThread, ThreadDirective) + case legacyGoroutine: + return None, token.NoPos, errorAt(fset, directive.Pos, "//%s is not supported; use %s", legacyGoroutine, GoroutineDirective) + default: + continue + } + if directive.Args != "" { + return None, token.NoPos, errorAt(fset, directive.Pos, "//%s does not accept arguments", directive.Name) + } + var err error + kind, pos, err = mergeAt(fset, kind, pos, next, directive.Pos) + if err != nil { + return None, token.NoPos, err + } + } + return kind, pos, nil +} + +// ValidateDoc rejects a locality directive outside a package-level var. +func ValidateDoc(fset *token.FileSet, doc *ast.CommentGroup) error { + kind, pos, err := FromDoc(fset, doc) + if err != nil { + return err + } + if kind != None { + return errorAt(fset, pos, "%s applies only to package-level var declarations", Directive(kind)) + } + return nil +} + +func mergeAt(fset *token.FileSet, a Kind, apos token.Pos, b Kind, bpos token.Pos) (Kind, token.Pos, error) { + merged, ok := Merge(a, b) + if !ok { + return None, token.NoPos, errorAt(fset, bpos, "%s and %s cannot apply to the same variable declaration", ThreadDirective, GoroutineDirective) + } + if b != None { + return merged, bpos, nil + } + return merged, apos, nil +} + +func hasDirective(doc *ast.CommentGroup, name string) bool { + for _, directive := range directive.ParseGroup(doc) { + if directive.Name == name { + return true + } + } + return false +} + +func errorAt(fset *token.FileSet, pos token.Pos, format string, args ...any) error { + msg := fmt.Sprintf(format, args...) + if fset == nil || pos == token.NoPos { + return fmt.Errorf("%s", msg) + } + return fmt.Errorf("%s: %s", fset.Position(pos), msg) +} From a233a966a540eac46cc53f3772a56d7285ffd6ed Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 14 Jul 2026 19:08:20 +0800 Subject: [PATCH 02/13] runtime: add stack-rooted locality contexts --- runtime/internal/runtime/local_context.go | 136 ++++++++++++++++++ .../runtime/local_context_baremetal.go | 23 +++ .../internal/runtime/local_context_stub.go | 35 +++++ runtime/internal/runtime/local_context_tls.go | 26 ++++ runtime/internal/runtime/local_initializer.go | 62 ++++++++ runtime/internal/runtime/z_default.go | 1 + 6 files changed, 283 insertions(+) create mode 100644 runtime/internal/runtime/local_context.go create mode 100644 runtime/internal/runtime/local_context_baremetal.go create mode 100644 runtime/internal/runtime/local_context_stub.go create mode 100644 runtime/internal/runtime/local_context_tls.go create mode 100644 runtime/internal/runtime/local_initializer.go diff --git a/runtime/internal/runtime/local_context.go b/runtime/internal/runtime/local_context.go new file mode 100644 index 0000000000..95f6226f8f --- /dev/null +++ b/runtime/internal/runtime/local_context.go @@ -0,0 +1,136 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import "unsafe" + +// LocalContext is rooted by the outer Go entry stack frame. The current +// one-thread-per-goroutine backend maps both logical locality kinds to this one +// physical package store. +type LocalContext struct { + blocks *localBlock +} + +type localBlock struct { + next *localBlock + key unsafe.Pointer +} + +func EnterLocalContext(ctx *LocalContext) uintptr { + previous := currentLocalContext + if previous == 0 { + if ctx == nil { + panic("runtime: nil local context") + } + currentLocalContext = uintptr(unsafe.Pointer(ctx)) + } + return previous +} + +func LeaveLocalContext(ctx *LocalContext, previous uintptr) { + if previous != 0 { + if currentLocalContext != previous { + panic("runtime: local context changed by nested entry") + } + return + } + if currentLocalContext != uintptr(unsafe.Pointer(ctx)) { + panic("runtime: leaving inactive local context") + } + currentLocalContext = 0 + releaseLocalBlocks(ctx) +} + +func leaveCurrentLocalContext() { + ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) + if ctx == nil { + return + } + currentLocalContext = 0 + releaseLocalBlocks(ctx) +} + +func releaseLocalBlocks(ctx *LocalContext) { + block := ctx.blocks + ctx.blocks = nil + for block != nil { + next := block.next + // Do not free block here: an address of a local variable may outlive its + // owner. Breaking the links lets the GC retain only escaped blocks. + block.next = nil + block = next + } +} + +// LocalPackage returns stable, zeroed storage for one package in the current +// physical owner. Repeated access to the most recently used package takes the +// head fast path; other accesses move the matching block to the front. +func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { + ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) + if ctx == nil { + panic("runtime: local variable accessed outside a Go entry context") + } + if key == nil { + panic("runtime: nil local package key") + } + if align == 0 || align&(align-1) != 0 { + panic("runtime: invalid local package alignment") + } + first := ctx.blocks + if first != nil && first.key == key { + return localBlockData(first, align) + } + var previous *localBlock + for block := first; block != nil; block = block.next { + if block.key == key { + previous.next = block.next + block.next = first + ctx.blocks = block + return localBlockData(block, align) + } + previous = block + } + block := newLocalBlock(key, size, align) + block.next = first + ctx.blocks = block + return localBlockData(block, align) +} + +func newLocalBlock(key unsafe.Pointer, size, align uintptr) *localBlock { + header := unsafe.Sizeof(localBlock{}) + padding := align - 1 + if size == 0 { + size = 1 + } + if header > ^uintptr(0)-padding || header+padding > ^uintptr(0)-size { + panic("runtime: local package size overflow") + } + block := (*localBlock)(AllocZ(header + padding + size)) + if block == nil { + panic("runtime: failed to allocate local package") + } + block.key = key + return block +} + +func localBlockData(block *localBlock, align uintptr) unsafe.Pointer { + padding := align - 1 + data := (uintptr(unsafe.Pointer(block)) + unsafe.Sizeof(localBlock{}) + padding) &^ padding + return unsafe.Pointer(data) +} diff --git a/runtime/internal/runtime/local_context_baremetal.go b/runtime/internal/runtime/local_context_baremetal.go new file mode 100644 index 0000000000..30557385ec --- /dev/null +++ b/runtime/internal/runtime/local_context_baremetal.go @@ -0,0 +1,23 @@ +//go:build llgo && baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +// Bare-metal runtimes have one execution context and do not introduce a native +// TLS relocation merely by linking the locality runtime support. +var currentLocalContext uintptr diff --git a/runtime/internal/runtime/local_context_stub.go b/runtime/internal/runtime/local_context_stub.go new file mode 100644 index 0000000000..ebd713e575 --- /dev/null +++ b/runtime/internal/runtime/local_context_stub.go @@ -0,0 +1,35 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import "unsafe" + +type LocalContext struct{} + +func EnterLocalContext(ctx *LocalContext) uintptr { + return 0 +} + +func LeaveLocalContext(ctx *LocalContext, previous uintptr) {} + +func leaveCurrentLocalContext() {} + +func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { + return AllocZ(size) +} diff --git a/runtime/internal/runtime/local_context_tls.go b/runtime/internal/runtime/local_context_tls.go new file mode 100644 index 0000000000..a06aa1cd93 --- /dev/null +++ b/runtime/internal/runtime/local_context_tls.go @@ -0,0 +1,26 @@ +//go:build llgo && !baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +// currentLocalContext is only an address cache. The context itself is rooted by +// the outer entry frame, so this native TLS slot intentionally has no GC-visible +// pointer type. +// +//llgo:tls +var currentLocalContext uintptr diff --git a/runtime/internal/runtime/local_initializer.go b/runtime/internal/runtime/local_initializer.go new file mode 100644 index 0000000000..58cb8a7fc0 --- /dev/null +++ b/runtime/internal/runtime/local_initializer.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import "unsafe" + +const ( + localInitUninitialized uint8 = iota + localInitInitializing + localInitReady + localInitFailed +) + +// EnsureLocalInitializer executes one package/locality dispatcher at most once +// in the current owner. Recursive access observes partial initialization; a +// recovered failure remains failed and re-panics on every later access. +func EnsureLocalInitializer(state *uint8, failureKey unsafe.Pointer, initialize func()) { + switch *state { + case localInitReady: + return + case localInitInitializing: + return + case localInitFailed: + panic(*localInitializerFailure(failureKey)) + case localInitUninitialized: + default: + panic("runtime: invalid local initializer state") + } + *state = localInitInitializing + completed := false + defer func() { + if completed { + return + } + value := recover() + *localInitializerFailure(failureKey) = value + *state = localInitFailed + panic(value) + }() + initialize() + completed = true + *state = localInitReady +} + +func localInitializerFailure(key unsafe.Pointer) *any { + var value any + return (*any)(LocalPackage(key, unsafe.Sizeof(value), unsafe.Alignof(value))) +} diff --git a/runtime/internal/runtime/z_default.go b/runtime/internal/runtime/z_default.go index 6ab61c0d66..401f8aaa17 100644 --- a/runtime/internal/runtime/z_default.go +++ b/runtime/internal/runtime/z_default.go @@ -40,6 +40,7 @@ func Rethrow(link *Defer) { fatal("no goroutines (main called runtime.Goexit) - deadlock!") c.Exit(2) } + leaveCurrentLocalContext() exitCurrentM() } } From 23352621ca482847d0f80fd297619772872d2013 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 14 Jul 2026 19:08:50 +0800 Subject: [PATCH 03/13] compiler: lower TLS and GLS package variables --- cl/compile.go | 40 +- cl/import.go | 89 ++-- cl/import_coverage_test.go | 83 ++- cl/locality.go | 106 ++++ cl/locality_lower.go | 419 +++++++++++++++ cl/locality_lower_test.go | 188 +++++++ cl/locality_test.go | 647 ++++++++++++++++++++++++ cl/static_init.go | 5 + internal/build/build.go | 47 +- internal/build/build_test.go | 129 ++++- internal/build/main_module.go | 8 + internal/build/main_module_test.go | 40 ++ internal/locality/layout/layout.go | 209 ++++++++ internal/locality/layout/layout_test.go | 166 ++++++ ssa/decl.go | 19 +- ssa/goroutine.go | 8 + ssa/goroutine_patch_test.go | 23 + ssa/local_context.go | 38 ++ ssa/locality.go | 199 ++++++++ ssa/locality_test.go | 159 ++++++ ssa/package.go | 17 +- ssa/ssa_test.go | 21 +- ssa/type.go | 5 + 23 files changed, 2615 insertions(+), 50 deletions(-) create mode 100644 cl/locality.go create mode 100644 cl/locality_lower.go create mode 100644 cl/locality_lower_test.go create mode 100644 cl/locality_test.go create mode 100644 internal/locality/layout/layout.go create mode 100644 internal/locality/layout/layout_test.go create mode 100644 ssa/local_context.go create mode 100644 ssa/locality.go create mode 100644 ssa/locality_test.go diff --git a/cl/compile.go b/cl/compile.go index 2257d3154e..8f99b9a0f5 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -210,6 +210,7 @@ type context struct { staticGlobalInits map[*ssa.Global]llssa.Expr staticInitStores map[*ssa.Store]none staticInitInstrs map[ssa.Instruction]none + locality localityLowering } func (p *context) rewriteValue(name string) (string, bool) { @@ -391,7 +392,10 @@ func (p *context) compileGlobal(pkg llssa.Package, gbl *ssa.Global) { return } dbgInstrln("==> NewVar", name, typ) - g := pkg.NewVar(name, typ, llssa.Background(vtype)) + g, skip := p.localityGlobalStorage(pkg, gbl, name, typ, llssa.Background(vtype)) + if skip { + return + } if p.tryEmbedGlobalInit(pkg, gbl, g, name) { return } @@ -601,12 +605,15 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun dbgSymsEnabled := enableDbgSyms && (f == nil || f.Origin() == nil) p.inits = append(p.inits, func() { oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark := p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark + oldLocalityFunction := p.locality.function p.fn = fn p.goFn = f p.callerFrameMark = llssa.Nil + p.locality.function = localityFunction{} p.state = state // restore pkgState when compiling funcBody defer func() { p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark = oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark + p.locality.function = oldLocalityFunction }() p.phis = nil if dbgSymsEnabled { @@ -624,6 +631,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun bodyPos := p.getFuncBodyPos(f) b.DebugFunction(fn, debugFunctionScope(f), pos, bodyPos) } + p.prepareExportedLocalContext(f) p.bvals = make(map[ssa.Value]llssa.Expr) p.methodNilDerefChecks = collectMethodNilDerefChecks(f) off := make([]int, len(f.Blocks)) @@ -832,6 +840,9 @@ func (p *context) debugParams(b llssa.Builder, f *ssa.Function) { } func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, doModInit bool) llssa.BasicBlock { + oldLocalBlock := p.locality.function.block + p.locality.function.block = block + defer func() { p.locality.function.block = oldLocalBlock }() var last int var pyModInit bool var prog = p.prog @@ -840,6 +851,9 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do var instrs = block.Instrs[n:] var ret = fn.Block(block.Index) b.SetBlock(ret) + if block.Index == 0 { + p.enterExportedLocalContext(b) + } if block.Index == 0 && p.shouldTrackCallerFrames() { p.pushCallerLocationFrame(b, block.Parent()) } @@ -852,6 +866,7 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do } if doModInit { + p.initializeLocalGuards(b) if p.state != pkgInPatch { p.applyEmbedInits(b) } @@ -1706,6 +1721,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { if p.shouldTrackCallerFrames() { p.popCallerLocationFrame(b) } + p.leaveExportedLocalContext(b) b.Return(results...) case *ssa.If: fn := p.fn @@ -1804,7 +1820,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr { if isCgoVar(varName) { p.cgoSymbols = append(p.cgoSymbols, val.Name()) } - if enableDbgSyms { + if enableDbgSyms && p.localityAllowsGlobalDebug(v) { pos := p.fset.Position(v.Pos()) b.DIGlobal(val, v.Name(), pos) } @@ -2068,6 +2084,15 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri pkg.Pkg = pkgTypes patch.Alt.Pkg = pkgTypes } + if err = ParsePkgSyntax(prog, pkgProg.Fset, pkgTypes, files); err != nil { + return nil, nil, err + } + if err = prog.ValidateLocalities(llssa.PathOf(pkgTypes)); err != nil { + return nil, nil, err + } + if err = validateLocalInitializers(prog, pkgTypes); err != nil { + return nil, nil, err + } if pkgPath == llssa.PkgRuntime { prog.SetRuntime(pkgTypes) } @@ -2174,6 +2199,17 @@ func processPkg(ctx *context, ret llssa.Package, pkg *ssa.Package) { sort.Slice(members, func(i, j int) bool { return members[i].name < members[j].name }) + localGlobals := make([]*ssa.Global, 0) + for _, m := range members { + global, ok := m.val.(*ssa.Global) + if !ok || isCgoFuncPtrVar(global.Name()) { + continue + } + localGlobals = append(localGlobals, global) + } + // Address accessors and replay guards must exist before any function body + // can reference a local package variable, regardless of member sort order. + ctx.prepareLocalVariables(ret, localGlobals) for _, m := range members { member := m.val diff --git a/cl/import.go b/cl/import.go index eb7a0e2815..5b18496eac 100644 --- a/cl/import.go +++ b/cl/import.go @@ -28,7 +28,9 @@ import ( "golang.org/x/tools/go/ssa" + "github.com/goplus/llgo/internal/directive" "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/locality" llssa "github.com/goplus/llgo/ssa" ) @@ -218,30 +220,6 @@ func (p *context) initFiles(pkgPath string, files []*ast.File, cPkg bool) { } } -// PreCollectLinknames scans syntax files before SSA compilation and populates -// prog.Linkname for package-level //go:linkname / //llgo:link declarations. -// It intentionally ignores //export because there is no package export context -// during the pre-collection phase. -func PreCollectLinknames(prog llssa.Program, pkgPath string, files []*ast.File) { - ctx := &context{prog: prog} - for _, file := range files { - for _, decl := range file.Decls { - switch decl := decl.(type) { - case *ast.FuncDecl: - fullName, inPkgName := astFuncName(pkgPath, decl) - ctx.processLinknameByDoc(decl.Doc, fullName, inPkgName, false, false) - case *ast.GenDecl: - if decl.Tok == token.VAR && len(decl.Specs) == 1 { - if names := decl.Specs[0].(*ast.ValueSpec).Names; len(names) == 1 { - inPkgName := names[0].Name - ctx.processLinknameByDoc(decl.Doc, pkgPath+"."+inPkgName, inPkgName, true, false) - } - } - } - } - } -} - // Collect skip names and skip other annotations, such as go: and llgo: // llgo:skip symbol1 symbol2 ... // llgo:skipall @@ -298,6 +276,21 @@ func (p *context) collectSkip(line string, prefix int) { } } +func collectLinknameByDoc(prog llssa.Program, doc *ast.CommentGroup, fullName, inPkgName string) { + directives := directive.ParseGroup(doc) + for n := len(directives) - 1; n >= 0; n-- { + directive := directives[n] + if directive.Name != "go:linkname" && directive.Name != "llgo:link" { + continue + } + fields := strings.Fields(directive.Args) + if len(fields) >= 2 && fields[0] == inPkgName { + prog.SetLinkname(fullName, strings.Join(fields[1:], " ")) + return + } + } +} + func (p *context) processLinknameByDoc(doc *ast.CommentGroup, fullName, inPkgName string, isVar, allowExport bool) bool { if doc != nil { for n := len(doc.List) - 1; n >= 0; n-- { @@ -722,6 +715,9 @@ func (p *context) varOf(b llssa.Builder, v *ssa.Global) llssa.Expr { } panic("unreachable") } + if local, ok := p.localVariableAddress(b, v, name); ok { + return local + } ret := pkg.VarOf(name) if ret == nil { ret = pkg.NewVar(name, p.patchType(v.Type()), llssa.Background(vtype)) @@ -754,24 +750,59 @@ func (p *context) initPyModule() { } } -// ParsePkgSyntax parses AST of a package to check package-level compiler directives. -func ParsePkgSyntax(prog llssa.Program, pkg *types.Package, files []*ast.File) { +// ParsePkgSyntax collects declaration directives in one syntax pass before SSA +// creation. Directives that need an LLVM package (such as //export) are applied +// later by initFiles. +func ParsePkgSyntax(prog llssa.Program, fset *token.FileSet, pkg *types.Package, files []*ast.File) error { + if pkg == nil { + return nil + } + if prog.PackageSyntaxParsed(pkg) { + return nil + } ctx := &context{prog: prog} pkgPath := llssa.PathOf(pkg) for _, file := range files { for _, decl := range file.Decls { switch decl := decl.(type) { case *ast.FuncDecl: - fullName, _ := astFuncName(pkgPath, decl) + if err := locality.ValidateDoc(fset, decl.Doc); err != nil { + return err + } + if err := locality.ValidateFuncBody(fset, decl.Body); err != nil { + return err + } + fullName, inPkgName := astFuncName(pkgPath, decl) + collectLinknameByDoc(prog, decl.Doc, fullName, inPkgName) ctx.processNoInterfaceByDoc(decl.Doc, fullName) case *ast.GenDecl: - switch decl.Tok { - case token.TYPE: + if decl.Tok == token.VAR { + if len(decl.Specs) == 1 { + if names := decl.Specs[0].(*ast.ValueSpec).Names; len(names) == 1 { + inPkgName := names[0].Name + collectLinknameByDoc(prog, decl.Doc, pkgPath+"."+inPkgName, inPkgName) + } + } + vars, err := locality.ScanPackageVar(fset, decl) + if err != nil { + return err + } + for _, variable := range vars { + prog.SetLocalityInfo(llssa.FullName(pkg, variable.Name), variable.Info) + } + continue + } + if err := locality.ValidateNonPackageVar(fset, decl); err != nil { + return err + } + if decl.Tok == token.TYPE { handleTypeDecl(prog, pkg, decl) } } } } + prog.MarkPackageSyntaxParsed(pkg) + return nil } func handleTypeDecl(prog llssa.Program, pkg *types.Package, decl *ast.GenDecl) { diff --git a/cl/import_coverage_test.go b/cl/import_coverage_test.go index 345693b29c..4afa5b0110 100644 --- a/cl/import_coverage_test.go +++ b/cl/import_coverage_test.go @@ -62,7 +62,9 @@ func (A) StackedHidden() {} } prog := llssa.NewProgram(nil) pkg := types.NewPackage("example.com/p", "p") - ParsePkgSyntax(prog, pkg, []*ast.File{file}) + if err := ParsePkgSyntax(prog, fset, pkg, []*ast.File{file}); err != nil { + t.Fatal(err) + } ctx := &context{prog: prog} ctx.processNoInterfaceByDoc(nil, "example.com/p.NilDoc") @@ -70,6 +72,59 @@ func (A) StackedHidden() {} {Text: "// not a directive"}, {Text: "//go:nointerface"}, }}, "example.com/p.NonDirectiveStops") + + if !prog.PackageSyntaxParsed(pkg) { + t.Fatal("package syntax was not marked as parsed") + } + badFile, err := parser.ParseFile(fset, "bad.go", "package p\n//llgo:tls\nfunc Bad() {}\n", parser.ParseComments) + if err != nil { + t.Fatal(err) + } + if err := ParsePkgSyntax(prog, fset, pkg, []*ast.File{badFile}); err != nil { + t.Fatalf("already parsed package was scanned again: %v", err) + } +} + +func TestParsePkgSyntaxReportsLocalityErrors(t *testing.T) { + prog := llssa.NewProgram(nil) + if err := ParsePkgSyntax(prog, nil, nil, nil); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + src string + want string + }{ + { + name: "function body", + src: "package p\nfunc f() {\n//llgo:gls\nvar value int\n_ = value\n}\n", + want: "package-level var", + }, + { + name: "package var", + src: "package p\n//llgo:tls extra\nvar Value int\n", + want: "does not accept arguments", + }, + { + name: "non-var declaration", + src: "package p\n//llgo:gls\nconst Value = 1\n", + want: "package-level var", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "p.go", test.src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + pkg := types.NewPackage("example.com/"+strings.ReplaceAll(test.name, " ", "-"), "p") + err = ParsePkgSyntax(llssa.NewProgram(nil), fset, pkg, []*ast.File{file}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ParsePkgSyntax error = %v, want %q", err, test.want) + } + }) + } } func TestPkgSymInfoAddSymAndInitLinknamesCoverage(t *testing.T) { @@ -166,13 +221,14 @@ func TestAstAndTypesFuncNameCoverage(t *testing.T) { } } -func TestPreCollectLinknames(t *testing.T) { +func TestParsePkgSyntaxCollectsLinknames(t *testing.T) { cases := []struct { name string directive string want string }{ {name: "go-linkname", directive: "//go:linkname Sigsetjmp C.sigsetjmp", want: "C.sigsetjmp"}, + {name: "go-linkname-tabs", directive: "//go:linkname\tSigsetjmp\tC.sigsetjmp", want: "C.sigsetjmp"}, {name: "llgo-linkname", directive: "//llgo:link Sigsetjmp C.sigsetjmp", want: "C.sigsetjmp"}, {name: "llgo-linkname-spaced", directive: "// llgo:link Sigsetjmp C.sigsetjmp", want: "C.sigsetjmp"}, } @@ -185,12 +241,33 @@ func TestPreCollectLinknames(t *testing.T) { t.Fatalf("ParseFile failed: %v", err) } prog := llssa.NewProgram(nil) - PreCollectLinknames(prog, llssa.PkgRuntime, []*ast.File{file}) + pkg := types.NewPackage(llssa.PkgRuntime, "runtime") + if err := ParsePkgSyntax(prog, fset, pkg, []*ast.File{file}); err != nil { + t.Fatal(err) + } if got, ok := prog.Linkname(llssa.PkgRuntime + ".Sigsetjmp"); !ok || got != tt.want { t.Fatalf("pre-collected linkname = (%q,%v), want (%q,%v)", got, ok, tt.want, true) } }) } + prog := llssa.NewProgram(nil) + collectLinknameByDoc(prog, &ast.CommentGroup{List: []*ast.Comment{{Text: "//go:linkname Other C.other"}}}, llssa.PkgRuntime+".Sigsetjmp", "Sigsetjmp") + if _, ok := prog.Linkname(llssa.PkgRuntime + ".Sigsetjmp"); ok { + t.Fatal("mismatched linkname was collected") + } +} + +func TestCollectLinknameByDocIgnoresOtherDirectives(t *testing.T) { + prog := llssa.NewProgram(nil) + doc := &ast.CommentGroup{List: []*ast.Comment{ + {Text: "//go:noinline"}, + {Text: "//llgo:tls"}, + }} + const fullName = "example.com/p.Value" + collectLinknameByDoc(prog, doc, fullName, "Value") + if _, ok := prog.Linkname(fullName); ok { + t.Fatal("non-link directives installed a linkname") + } } func TestBoolToUint8InvalidArgs(t *testing.T) { diff --git a/cl/locality.go b/cl/locality.go new file mode 100644 index 0000000000..0961685f30 --- /dev/null +++ b/cl/locality.go @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "strings" + + "github.com/goplus/llgo/internal/locality" + localitylayout "github.com/goplus/llgo/internal/locality/layout" + llssa "github.com/goplus/llgo/ssa" +) + +// PrepareLocalVariables extracts replayable initializer helpers before Go SSA +// construction, then records the storage strategy selected by the independent +// package-layout planner. +func PrepareLocalVariables(prog llssa.Program, fset *token.FileSet, pkg *types.Package, info *types.Info, files []*ast.File) error { + if pkg == nil || info == nil { + return nil + } + path := llssa.PathOf(pkg) + prepared, err := locality.Prepare(fset, path, pkg, info, files, packageLocalities(prog, path)) + if err != nil { + return err + } + for name, local := range prepared { + prog.SetLocalityInfo(llssa.FullName(pkg, name), local) + } + for fullName, local := range prog.PackageLocalities(path) { + name := strings.TrimPrefix(fullName, path+".") + object, _ := pkg.Scope().Lookup(name).(*types.Var) + if object == nil { + return fmt.Errorf("locality layout: package %s has no variable %s", path, name) + } + canonical, _, _, err := prog.ResolveLocality(fullName) + if err != nil { + return err + } + if canonical != fullName && local.HasInitializer { + return fmt.Errorf("locality layout: linkname alias %s cannot have an initializer", fullName) + } + prog.SetLocalStorage(fullName, localitylayout.StorageForType(object.Type())) + } + _, err = planLocalPackage(prog, pkg) + return err +} + +func validateLocalInitializers(prog llssa.Program, pkg *types.Package) error { + return locality.ValidatePrepared(llssa.PathOf(pkg), packageLocalities(prog, llssa.PathOf(pkg))) +} + +func packageLocalities(prog llssa.Program, pkgPath string) map[string]locality.Info { + prefix := pkgPath + "." + ret := make(map[string]locality.Info) + for name, info := range prog.PackageLocalities(pkgPath) { + ret[strings.TrimPrefix(name, prefix)] = info.Info + } + return ret +} + +func planLocalPackage(prog llssa.Program, pkg *types.Package) (localitylayout.Package, error) { + if pkg == nil { + return localitylayout.Package{}, nil + } + path := llssa.PathOf(pkg) + prefix := path + "." + decls := prog.PackageLocalities(path) + input := make([]localitylayout.Declaration, 0, len(decls)) + for fullName := range decls { + canonical, info, _, err := prog.ResolveLocality(fullName) + if err != nil { + return localitylayout.Package{}, err + } + if canonical != fullName { + target, targetOK := prog.VariableLocality(canonical) + if !targetOK || target.Locality == locality.None { + return localitylayout.Package{}, fmt.Errorf("locality layout: linkname target %s for %s is not a local variable", canonical, fullName) + } + continue + } + name := strings.TrimPrefix(fullName, prefix) + object, _ := pkg.Scope().Lookup(name).(*types.Var) + if object == nil { + return localitylayout.Package{}, fmt.Errorf("locality layout: package %s has no variable %s", path, name) + } + input = append(input, localitylayout.Declaration{Name: fullName, Type: object.Type(), Info: info.Info}) + } + return localitylayout.Plan(path, input) +} diff --git a/cl/locality_lower.go b/cl/locality_lower.go new file mode 100644 index 0000000000..10b6cd4cee --- /dev/null +++ b/cl/locality_lower.go @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl + +import ( + "fmt" + "go/token" + "go/types" + "strings" + + "github.com/goplus/llgo/internal/locality" + localitylayout "github.com/goplus/llgo/internal/locality/layout" + llssa "github.com/goplus/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +const localInitReady = 2 + +type localVariable struct { + planned localitylayout.Variable + owner *localPackage +} + +type localPackage struct { + plan localitylayout.Package + typ llssa.Type + blockFunc llssa.Function + direct map[string]llssa.Global + init map[locality.Kind]*localInitializer +} + +type localInitializer struct { + guard llssa.Global + failureKey llssa.Global + dispatch llssa.Function + ensure llssa.Function +} + +type localBaseCacheKey struct { + block *ssa.BasicBlock + owner *localPackage +} + +type localEnsureCacheKey struct { + block *ssa.BasicBlock + owner *localPackage + kind locality.Kind +} + +// localityLowering owns all compiler state for TLS/GLS lowering. Only this +// value is embedded in the general compiler context. +type localityLowering struct { + packages map[string]*localPackage + variables map[*ssa.Global]*localVariable + function localityFunction +} + +type localityFunction struct { + block *ssa.BasicBlock + packageBases map[localBaseCacheKey]llssa.Expr + packageEnsures map[localEnsureCacheKey]bool + entry *localEntryContext +} + +type localEntryContext struct { + context llssa.Expr + previous llssa.Expr + entered bool +} + +func (p *context) prepareExportedLocalContext(f *ssa.Function) { + if !p.prog.NeedsLocalContext() || f == nil || f.Pkg == nil { + return + } + fullName := funcName(f.Pkg.Pkg, f, false) + if _, exported := p.pkg.ExportFuncs()[fullName]; !exported { + return + } + p.locality.function.entry = &localEntryContext{} +} + +func (p *context) enterExportedLocalContext(b llssa.Builder) { + entry := p.locality.function.entry + if entry == nil || entry.entered { + return + } + context, previous := b.EnterLocalContext() + entry.context = context + entry.previous = previous + entry.entered = true +} + +func (p *context) leaveExportedLocalContext(b llssa.Builder) { + entry := p.locality.function.entry + if entry != nil && entry.entered { + b.LeaveLocalContext(entry.context, entry.previous) + } +} + +func (p *context) prepareLocalVariables(pkg llssa.Package, globals []*ssa.Global) { + _, err := p.localPackageFor(p.goTyps, pkg, true) + if err != nil { + panic(err) + } + if p.locality.variables == nil { + p.locality.variables = make(map[*ssa.Global]*localVariable) + } + for _, global := range globals { + variable, ok, err := p.localVariableFor(pkg, global, true) + if err != nil { + panic(err) + } + if !ok { + continue + } + p.locality.variables[global] = variable + } +} + +// localVariableFor resolves one Go SSA global to the canonical package plan. +// Both local definitions and imported references use this path so linkname and +// layout validation cannot diverge between the two lowering cases. +func (p *context) localVariableFor(pkg llssa.Package, global *ssa.Global, defineCurrent bool) (*localVariable, bool, error) { + fullName := llssa.FullName(global.Pkg.Pkg, global.Name()) + canonical, info, ok, err := p.prog.ResolveLocality(fullName) + if err != nil { + return nil, false, err + } + if !ok || info.Locality == locality.None { + return nil, false, nil + } + typesPkg := p.localTypesPackage(canonical) + if typesPkg == nil { + return nil, false, fmt.Errorf("missing types package for local variable %s", canonical) + } + owner, err := p.localPackageFor(typesPkg, pkg, defineCurrent && typesPkg == p.goTyps) + if err != nil { + return nil, false, err + } + planned, ok := owner.plan.Lookup(canonical) + if !ok { + return nil, false, fmt.Errorf("missing locality layout for %s", canonical) + } + return &localVariable{planned: planned, owner: owner}, true, nil +} + +func (p *context) localityGlobalStorage(pkg llssa.Package, global *ssa.Global, name string, typ types.Type, bg llssa.Background) (llssa.Global, bool) { + info, ok := p.resolveLocality(llssa.FullName(global.Pkg.Pkg, global.Name())) + if !ok || info.Locality == locality.None { + return pkg.NewVar(name, typ, bg), false + } + variable := p.locality.variables[global] + if variable == nil { + panic(fmt.Sprintf("missing locality layout for %s", name)) + } + if variable.planned.Storage == localitylayout.StoragePackage { + return nil, true + } + return variable.owner.direct[variable.planned.Name], false +} + +func (p *context) localityAllowsGlobalDebug(global *ssa.Global) bool { + variable := p.locality.variables[global] + return variable == nil || variable.planned.Storage == localitylayout.StorageNativeTLS +} + +func (p *context) localTypesPackage(fullName string) *types.Package { + matches := func(pkg *types.Package) bool { + if pkg == nil { + return false + } + prefix := llssa.PathOf(pkg) + "." + if !strings.HasPrefix(fullName, prefix) { + return false + } + name := strings.TrimPrefix(fullName, prefix) + _, ok := pkg.Scope().Lookup(name).(*types.Var) + return ok + } + if matches(p.goTyps) { + return p.goTyps + } + if p.goProg != nil { + for _, pkg := range p.goProg.AllPackages() { + if pkg != nil && matches(pkg.Pkg) { + return pkg.Pkg + } + } + } + for pkg := range p.loaded { + if matches(pkg) { + return pkg + } + } + return nil +} + +func (p *context) localPackageFor(typesPkg *types.Package, pkg llssa.Package, define bool) (*localPackage, error) { + if typesPkg == nil { + return nil, nil + } + path := llssa.PathOf(typesPkg) + if owner := p.locality.packages[path]; owner != nil { + return owner, nil + } + plan, err := planLocalPackage(p.prog, typesPkg) + if err != nil { + return nil, err + } + if len(plan.Variables) == 0 { + return nil, nil + } + if p.locality.packages == nil { + p.locality.packages = make(map[string]*localPackage) + } + owner := &localPackage{ + plan: plan, + direct: make(map[string]llssa.Global), + init: make(map[locality.Kind]*localInitializer), + } + p.locality.packages[path] = owner + p.buildLocalPackage(pkg, owner, define) + return owner, nil +} + +func (p *context) buildLocalPackage(pkg llssa.Package, owner *localPackage, define bool) { + for _, variable := range owner.plan.Variables { + if variable.Storage != localitylayout.StorageNativeTLS { + continue + } + typ := types.NewPointer(p.patchType(variable.Type)) + global := pkg.NewThreadLocalVar(variable.Name, typ, llssa.InGo) + owner.direct[variable.Name] = global + } + if len(owner.plan.Block) != 0 { + fields := make([]*types.Var, len(owner.plan.Block)) + for index, variable := range owner.plan.Block { + fields[index] = types.NewField(token.NoPos, nil, fmt.Sprintf("v%d", index), p.patchType(variable.Type), false) + } + structType := types.NewStruct(fields, nil) + owner.typ = p.prog.Type(structType, llssa.InGo) + key := pkg.NewVar(localitylayout.BlockKeyName(owner.plan.Path), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) + if define { + key.InitNil() + } + result := types.NewPointer(structType) + owner.blockFunc = pkg.NewFunc(localitylayout.BlockName(owner.plan.Path), noArgResultSignature(result), llssa.InGo) + owner.blockFunc.Inline(llssa.AlwaysInline) + if define && !owner.blockFunc.HasBody() { + b := owner.blockFunc.MakeBody(1) + raw := b.Call( + pkg.RuntimeFunc("LocalPackage"), + b.Convert(p.prog.VoidPtr(), key.Expr), + p.prog.IntVal(p.prog.SizeOf(owner.typ), p.prog.Uintptr()), + p.prog.IntVal(p.prog.AlignOf(owner.typ), p.prog.Uintptr()), + ) + b.Return(b.Convert(p.prog.Pointer(owner.typ), raw)) + b.EndBuild() + } + } + for _, kind := range []locality.Kind{locality.Thread, locality.Goroutine} { + initializers := owner.plan.Initializers(kind) + if len(initializers) == 0 { + continue + } + owner.init[kind] = p.buildLocalInitializer(pkg, owner, kind, initializers, define) + } +} + +func (p *context) buildLocalInitializer(pkg llssa.Package, owner *localPackage, kind locality.Kind, initializers []localitylayout.Initializer, define bool) *localInitializer { + ret := &localInitializer{} + ret.guard = pkg.NewThreadLocalVar(localitylayout.GuardName(owner.plan.Path, kind), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) + ret.failureKey = pkg.NewVar(localitylayout.FailureKeyName(owner.plan.Path, kind), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) + ret.dispatch = pkg.NewFunc(localitylayout.InitName(owner.plan.Path, kind), llssa.NoArgsNoRet, llssa.InGo) + ret.ensure = pkg.NewFunc(localitylayout.EnsureName(owner.plan.Path, kind), llssa.NoArgsNoRet, llssa.InGo) + ret.ensure.Inline(llssa.AlwaysInline) + if !define { + return ret + } + ret.guard.InitNil() + ret.failureKey.InitNil() + if !ret.dispatch.HasBody() { + b := ret.dispatch.MakeBody(1) + for _, initializer := range initializers { + helper := pkg.NewFunc(initializer.Name, llssa.NoArgsNoRet, llssa.InGo) + b.Call(helper.Expr) + } + b.Return() + b.EndBuild() + } + if !ret.ensure.HasBody() { + b := ret.ensure.MakeBody(3) + ready := b.BinOp(token.EQL, b.Load(ret.guard.Expr), p.prog.IntVal(localInitReady, p.prog.Byte())) + b.If(ready, ret.ensure.Block(2), ret.ensure.Block(1)) + b.SetBlock(ret.ensure.Block(1)) + closure := b.MakeClosure(ret.dispatch.Expr, nil) + b.Call( + pkg.RuntimeFunc("EnsureLocalInitializer"), + ret.guard.Expr, + b.Convert(p.prog.VoidPtr(), ret.failureKey.Expr), + closure, + ) + b.Jump(ret.ensure.Block(2)) + b.SetBlock(ret.ensure.Block(2)) + b.Return() + b.EndBuild() + } + return ret +} + +func noArgResultSignature(result types.Type) *types.Signature { + results := types.NewTuple(types.NewVar(token.NoPos, nil, "", result)) + return types.NewSignatureType(nil, nil, nil, nil, results, false) +} + +func (p *context) localVariableAddr(b llssa.Builder, v *ssa.Global, info llssa.VariableLocality, name string) llssa.Expr { + variable := p.locality.variables[v] + if variable == nil { + var ok bool + var err error + variable, ok, err = p.localVariableFor(p.pkg, v, false) + if err != nil { + panic(err) + } + if !ok { + panic(fmt.Sprintf("missing locality metadata for %s", name)) + } + p.locality.variables[v] = variable + } + p.ensureLocalInitializer(b, variable.owner, info.Locality) + if variable.planned.Storage == localitylayout.StorageNativeTLS { + direct := variable.owner.direct[variable.planned.Name] + if direct == nil { + panic(fmt.Sprintf("missing native TLS storage for %s", name)) + } + return direct.Expr + } + base := p.localPackageBase(b, variable.owner) + return b.FieldAddr(base, variable.planned.Field) +} + +func (p *context) localVariableAddress(b llssa.Builder, variable *ssa.Global, name string) (llssa.Expr, bool) { + info, ok := p.resolveLocality(llssa.FullName(variable.Pkg.Pkg, variable.Name())) + if !ok || info.Locality == locality.None { + return llssa.Expr{}, false + } + return p.localVariableAddr(b, variable, info, name), true +} + +func (p *context) resolveLocality(name string) (llssa.VariableLocality, bool) { + _, info, ok, err := p.prog.ResolveLocality(name) + if err != nil { + panic(err) + } + return info, ok +} + +func (p *context) localPackageBase(b llssa.Builder, owner *localPackage) llssa.Expr { + state := &p.locality.function + for block := state.block; block != nil; block = block.Idom() { + if base, ok := state.packageBases[localBaseCacheKey{block: block, owner: owner}]; ok { + return base + } + } + base := b.Call(owner.blockFunc.Expr) + if state.block != nil { + if state.packageBases == nil { + state.packageBases = make(map[localBaseCacheKey]llssa.Expr) + } + state.packageBases[localBaseCacheKey{block: state.block, owner: owner}] = base + } + return base +} + +func (p *context) ensureLocalInitializer(b llssa.Builder, owner *localPackage, kind locality.Kind) { + initializer := owner.init[kind] + if initializer == nil { + return + } + state := &p.locality.function + for block := state.block; block != nil; block = block.Idom() { + if state.packageEnsures[localEnsureCacheKey{block: block, owner: owner, kind: kind}] { + return + } + } + b.Call(initializer.ensure.Expr) + if state.block != nil { + if state.packageEnsures == nil { + state.packageEnsures = make(map[localEnsureCacheKey]bool) + } + state.packageEnsures[localEnsureCacheKey{block: state.block, owner: owner, kind: kind}] = true + } +} + +func (p *context) initializeLocalGuards(b llssa.Builder) { + owner := p.locality.packages[llssa.PathOf(p.goTyps)] + if owner == nil { + return + } + for _, kind := range []locality.Kind{locality.Thread, locality.Goroutine} { + if initializer := owner.init[kind]; initializer != nil { + b.Store(initializer.guard.Expr, p.prog.IntVal(localInitReady, p.prog.Byte())) + } + } +} diff --git a/cl/locality_lower_test.go b/cl/locality_lower_test.go new file mode 100644 index 0000000000..7610d70c13 --- /dev/null +++ b/cl/locality_lower_test.go @@ -0,0 +1,188 @@ +package cl + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/goplus/llgo/internal/locality" + localitylayout "github.com/goplus/llgo/internal/locality/layout" + llssa "github.com/goplus/llgo/ssa" + "github.com/goplus/llgo/ssa/ssatest" + "golang.org/x/tools/go/ssa" +) + +func localitySSAGlobal(t *testing.T, path string) (*types.Package, *ssa.Global) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", "package locality\nvar Value int\n", 0) + if err != nil { + t.Fatal(err) + } + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check(path, fset, []*ast.File{file}, info) + if err != nil { + t.Fatal(err) + } + goProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions) + ssaPkg := goProg.CreatePackage(pkg, []*ast.File{file}, info, true) + global, ok := ssaPkg.Members["Value"].(*ssa.Global) + if !ok { + t.Fatalf("Value SSA member = %T", ssaPkg.Members["Value"]) + } + return pkg, global +} + +func TestLocalityLoweringResolution(t *testing.T) { + typesPkg, global := localitySSAGlobal(t, "example.com/lowering") + prog := ssatest.NewProgram(t, nil) + llvmPkg := prog.NewPackage(typesPkg.Name(), typesPkg.Path()) + ctx := &context{prog: prog, pkg: llvmPkg, goTyps: typesPkg} + + if variable, ok, err := ctx.localVariableFor(llvmPkg, global, true); err != nil || ok || variable != nil { + t.Fatalf("ordinary localVariableFor = %+v, %v, %v", variable, ok, err) + } + name := llssa.FullName(typesPkg, global.Name()) + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + prog.SetLocalStorage(name, llssa.LocalStorageNativeTLS) + variable, ok, err := ctx.localVariableFor(llvmPkg, global, true) + if err != nil || !ok || variable.planned.Storage != localitylayout.StorageNativeTLS { + t.Fatalf("local localVariableFor = %+v, %v, %v", variable, ok, err) + } + ctx.locality.variables = map[*ssa.Global]*localVariable{global: variable} + if !ctx.localityAllowsGlobalDebug(global) { + t.Fatal("native TLS was hidden from global debug info") + } + variable.planned.Storage = localitylayout.StoragePackage + if ctx.localityAllowsGlobalDebug(global) { + t.Fatal("package storage was exposed as a fixed debug global") + } + if !ctx.localityAllowsGlobalDebug(new(ssa.Global)) { + t.Fatal("ordinary global was hidden from debug info") + } + if owner, err := ctx.localPackageFor(typesPkg, llvmPkg, true); err != nil || owner != variable.owner { + t.Fatalf("cached localPackageFor = %p, %v; want %p", owner, err, variable.owner) + } + + loaded := types.NewPackage("example.com/loaded", "loaded") + loaded.Scope().Insert(types.NewVar(token.NoPos, loaded, "Value", types.Typ[types.Int])) + ctx.loaded = map[*types.Package]*pkgInfo{loaded: {kind: PkgDeclOnly}} + if got := ctx.localTypesPackage("example.com/loaded.Value"); got != loaded { + t.Fatalf("loaded localTypesPackage = %v, want %v", got, loaded) + } + if got := (&context{}).localTypesPackage("example.com/missing.Value"); got != nil { + t.Fatalf("missing localTypesPackage = %v", got) + } + if owner, err := ctx.localPackageFor(nil, llvmPkg, false); err != nil || owner != nil { + t.Fatalf("nil localPackageFor = %v, %v", owner, err) + } + empty := types.NewPackage("example.com/empty", "empty") + if owner, err := ctx.localPackageFor(empty, llvmPkg, false); err != nil || owner != nil { + t.Fatalf("empty localPackageFor = %v, %v", owner, err) + } +} + +func TestLocalityLoweringDiagnostics(t *testing.T) { + typesPkg, global := localitySSAGlobal(t, "example.com/diagnostic") + name := llssa.FullName(typesPkg, global.Name()) + newProgram := func() llssa.Program { + prog := ssatest.NewProgram(t, nil) + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + return prog + } + + t.Run("missing types package", func(t *testing.T) { + ctx := &context{prog: newProgram()} + if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "missing types package") { + t.Fatalf("localVariableFor error = %v", err) + } + }) + + t.Run("invalid plan", func(t *testing.T) { + prog := newProgram() + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal, HasInitializer: true}) + ctx := &context{prog: prog, goTyps: typesPkg} + if _, err := ctx.localPackageFor(typesPkg, nil, false); err == nil || !strings.Contains(err.Error(), "inconsistent initializer metadata") { + t.Fatalf("localPackageFor error = %v", err) + } + if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "inconsistent initializer metadata") { + t.Fatalf("localVariableFor error = %v", err) + } + }) + + t.Run("stale plan", func(t *testing.T) { + prog := newProgram() + ctx := &context{prog: prog, goTyps: typesPkg} + ctx.locality.packages = map[string]*localPackage{ + typesPkg.Path(): {plan: localitylayout.Package{Path: typesPkg.Path()}}, + } + if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "missing locality layout") { + t.Fatalf("localVariableFor error = %v", err) + } + }) + + t.Run("linkname cycle", func(t *testing.T) { + prog := newProgram() + other := typesPkg.Path() + ".Other" + prog.SetLinkname(name, other) + prog.SetLinkname(other, name) + ctx := &context{prog: prog, goTyps: typesPkg} + if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + t.Fatalf("localVariableFor error = %v", err) + } + assertLocalityPanic(t, "resolveLocality", func() { ctx.resolveLocality(name) }) + assertLocalityPanic(t, "prepareLocalVariables", func() { ctx.prepareLocalVariables(nil, nil) }) + assertLocalityPanic(t, "localVariableAddr", func() { + ctx.localVariableAddr(nil, global, llssa.VariableLocality{Info: llssa.LocalityInfo{Locality: llssa.ThreadLocal}}, name) + }) + }) + + t.Run("imported metadata error", func(t *testing.T) { + current := types.NewPackage("example.com/current", "current") + prog := newProgram() + other := typesPkg.Path() + ".Other" + prog.SetLinkname(name, other) + prog.SetLinkname(other, name) + ctx := &context{prog: prog, goTyps: current} + assertLocalityPanic(t, "prepare imported local", func() { ctx.prepareLocalVariables(nil, []*ssa.Global{global}) }) + }) + + t.Run("missing prepared layout", func(t *testing.T) { + ctx := &context{prog: newProgram()} + assertLocalityPanic(t, "localityGlobalStorage", func() { + ctx.localityGlobalStorage(nil, global, name, global.Type(), llssa.InGo) + }) + }) + + t.Run("missing address metadata", func(t *testing.T) { + ctx := &context{prog: ssatest.NewProgram(t, nil)} + assertLocalityPanic(t, "localVariableAddr", func() { + ctx.localVariableAddr(nil, global, llssa.VariableLocality{Info: llssa.LocalityInfo{Locality: llssa.ThreadLocal}}, name) + }) + }) + + t.Run("missing native storage", func(t *testing.T) { + ctx := &context{locality: localityLowering{variables: map[*ssa.Global]*localVariable{ + global: { + planned: localitylayout.Variable{Declaration: localitylayout.Declaration{Name: name}, Storage: localitylayout.StorageNativeTLS}, + owner: &localPackage{direct: map[string]llssa.Global{}, init: map[locality.Kind]*localInitializer{}}, + }, + }}} + assertLocalityPanic(t, "localVariableAddr", func() { + ctx.localVariableAddr(nil, global, llssa.VariableLocality{Info: llssa.LocalityInfo{Locality: llssa.ThreadLocal}}, name) + }) + }) +} + +func assertLocalityPanic(t *testing.T, name string, fn func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Fatalf("%s did not panic", name) + } + }() + fn() +} diff --git a/cl/locality_test.go b/cl/locality_test.go new file mode 100644 index 0000000000..081e8b35e7 --- /dev/null +++ b/cl/locality_test.go @@ -0,0 +1,647 @@ +package cl + +import ( + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "runtime" + "strings" + "testing" + + llssa "github.com/goplus/llgo/ssa" + "github.com/goplus/llgo/ssa/ssatest" + "golang.org/x/tools/go/ssa" +) + +func compileLocalitySource(t *testing.T, src string) (llssa.Program, string) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := newLocalityTypeInfo() + imp := importer.Default() + pkg, err := (&types.Config{Importer: imp}).Check("example.com/locality", fset, files, info) + if err != nil { + t.Fatal(err) + } + prog := ssatest.NewProgramEx(t, nil, imp) + prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) + prog.SetRuntime(localityRuntimePackage()) + if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, fset, pkg, info, files); err != nil { + t.Fatal(err) + } + goProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions) + ssaPkg := goProg.CreatePackage(pkg, files, info, true) + ssaPkg.Build() + compiled, err := NewPackage(prog, ssaPkg, files) + if err != nil { + t.Fatal(err) + } + return prog, compiled.String() +} + +func newLocalityTypeInfo() *types.Info { + return &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + Instances: make(map[*ast.Ident]types.Instance), + } +} + +func localityRuntimePackage() *types.Package { + pkg := types.NewPackage(llssa.PkgRuntime, "runtime") + localContextName := types.NewTypeName(token.NoPos, pkg, "LocalContext", nil) + localContext := types.NewNamed(localContextName, types.NewStruct(nil, nil), nil) + pkg.Scope().Insert(localContextName) + + localPackageParams := types.NewTuple( + types.NewParam(token.NoPos, pkg, "key", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, pkg, "size", types.Typ[types.Uintptr]), + types.NewParam(token.NoPos, pkg, "align", types.Typ[types.Uintptr]), + ) + localPackageResults := types.NewTuple(types.NewParam(token.NoPos, pkg, "", types.Typ[types.UnsafePointer])) + pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "LocalPackage", types.NewSignatureType(nil, nil, nil, localPackageParams, localPackageResults, false))) + + callback := types.NewSignatureType(nil, nil, nil, nil, nil, false) + ensureParams := types.NewTuple( + types.NewParam(token.NoPos, pkg, "state", types.NewPointer(types.Typ[types.Uint8])), + types.NewParam(token.NoPos, pkg, "failureKey", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, pkg, "initialize", callback), + ) + pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "EnsureLocalInitializer", types.NewSignatureType(nil, nil, nil, ensureParams, nil, false))) + + contextPointer := types.NewPointer(localContext) + enterParams := types.NewTuple(types.NewParam(token.NoPos, pkg, "ctx", contextPointer)) + enterResults := types.NewTuple(types.NewParam(token.NoPos, pkg, "previous", types.Typ[types.Uintptr])) + pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "EnterLocalContext", types.NewSignatureType(nil, nil, nil, enterParams, enterResults, false))) + leaveParams := types.NewTuple( + types.NewParam(token.NoPos, pkg, "ctx", contextPointer), + types.NewParam(token.NoPos, pkg, "previous", types.Typ[types.Uintptr]), + ) + pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "LeaveLocalContext", types.NewSignatureType(nil, nil, nil, leaveParams, nil, false))) + return pkg +} + +func llvmFunction(t *testing.T, ir, name string) string { + t.Helper() + markerAt := strings.Index(ir, `@"`+name+`"(`) + if markerAt < 0 { + markerAt = strings.Index(ir, `@`+name+`(`) + } + if markerAt < 0 { + t.Fatalf("function %s not found:\n%s", name, ir) + } + start := strings.LastIndex(ir[:markerAt], "define ") + if start < 0 { + t.Fatalf("definition for %s not found", name) + } + end := strings.Index(ir[markerAt:], "\n}") + if end < 0 { + t.Fatalf("end of %s not found", name) + } + return ir[start : markerAt+end+2] +} + +func TestLocalityPlansNativeTLSAndSharedPointerBlock(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality + +var backing int +func scalar() int { return 42 } +func pointer() *int { return &backing } + +//llgo:tls +var TLSScalar = scalar() +//llgo:tls +var TLSPointer = pointer() +//llgo:gls +var GLSScalar = scalar() +//llgo:gls +var GLSPointer = pointer() + +func values() (int, *int, int, *int) { + return TLSScalar, TLSPointer, GLSScalar, GLSPointer +} +`) + + checks := map[string]struct { + kind llssa.Locality + storage llssa.LocalStorage + }{ + "TLSScalar": {llssa.ThreadLocal, llssa.LocalStorageNativeTLS}, + "TLSPointer": {llssa.ThreadLocal, llssa.LocalStoragePackage}, + "GLSScalar": {llssa.GoroutineLocal, llssa.LocalStorageNativeTLS}, + "GLSPointer": {llssa.GoroutineLocal, llssa.LocalStoragePackage}, + } + for name, want := range checks { + got, ok := prog.VariableLocality("example.com/locality." + name) + if !ok || got.Locality != want.kind || got.LocalStorage != want.storage { + t.Fatalf("%s metadata = %+v, %v", name, got, ok) + } + } + for _, name := range []string{"TLSScalar", "GLSScalar"} { + if !strings.Contains(ir, `@"example.com/locality.`+name+`" = thread_local global i64`) { + t.Fatalf("%s is not native TLS:\n%s", name, ir) + } + } + for _, name := range []string{"TLSPointer", "GLSPointer"} { + if strings.Contains(ir, `@"example.com/locality.`+name+`" = thread_local`) { + t.Fatalf("%s retained a pointer-bearing TLS global:\n%s", name, ir) + } + } + if got := strings.Count(ir, `@"example.com/locality.__llgo_local_key" =`); got != 1 { + t.Fatalf("package block keys = %d, want 1:\n%s", got, ir) + } + if got := strings.Count(ir, `call ptr @"github.com/goplus/llgo/runtime/internal/runtime.LocalPackage"`); got != 1 { + t.Fatalf("LocalPackage calls = %d, want one accessor definition:\n%s", got, ir) + } + values := llvmFunction(t, ir, "example.com/locality.values") + if got := strings.Count(values, `call ptr @"example.com/locality.__llgo_local_block"()`); got != 1 { + t.Fatalf("values package-base calls = %d, want 1:\n%s", got, values) + } + if got := strings.Count(values, `call void @"example.com/locality.__llgo_tls_init$ensure"()`); got != 1 { + t.Fatalf("values TLS ensure calls = %d, want 1:\n%s", got, values) + } + if got := strings.Count(values, `call void @"example.com/locality.__llgo_gls_init$ensure"()`); got != 1 { + t.Fatalf("values GLS ensure calls = %d, want 1:\n%s", got, values) + } + if !prog.NeedsLocalContext() { + t.Fatal("pointer-bearing local variables did not enable a local context") + } +} + +func TestLocalityInitializersPreserveGoOrderPerKind(t *testing.T) { + _, ir := compileLocalitySource(t, `package locality + +func mark(value int) int { return value } +//llgo:tls +var T0 = mark(0) +//llgo:gls +var G0 = mark(1) +//llgo:tls +var T1 = mark(2) +func values() (int, int, int) { return T0, T1, G0 } +`) + tls := llvmFunction(t, ir, "example.com/locality.__llgo_tls_init") + gls := llvmFunction(t, ir, "example.com/locality.__llgo_gls_init") + first := strings.Index(tls, `__llgo_local_init_0`) + second := strings.Index(tls, `__llgo_local_init_2`) + if first < 0 || second < first || strings.Contains(tls, `__llgo_local_init_1`) { + t.Fatalf("TLS dispatcher order is wrong:\n%s", tls) + } + if !strings.Contains(gls, `__llgo_local_init_1`) || strings.Contains(gls, `__llgo_local_init_0`) || strings.Contains(gls, `__llgo_local_init_2`) { + t.Fatalf("GLS dispatcher contains the wrong helpers:\n%s", gls) + } + initBody := llvmFunction(t, ir, "example.com/locality.init") + for _, guard := range []string{"__llgo_tls_init$guard", "__llgo_gls_init$guard"} { + if !strings.Contains(initBody, `store i8 2, ptr @"example.com/locality.`+guard+`"`) { + t.Fatalf("package init does not mark %s ready:\n%s", guard, initBody) + } + } +} + +func TestDirectInitializerStillRequiresFailureContext(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality +func value() int { return 1 } +//llgo:tls +var Value = value() +func get() int { return Value } +`) + if !prog.NeedsLocalContext() { + t.Fatal("initializer failure storage did not enable a local context") + } + if strings.Contains(ir, `__llgo_local_block`) { + t.Fatalf("pointer-free package unexpectedly has a value block:\n%s", ir) + } + if !strings.Contains(ir, `@"example.com/locality.Value" = thread_local global i64`) || !strings.Contains(ir, `EnsureLocalInitializer`) { + t.Fatalf("direct initializer lowering is incomplete:\n%s", ir) + } +} + +func TestZeroValueDirectLocalsNeedNoContext(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality +//llgo:tls +var T int +//llgo:gls +var G uintptr +func values() (int, uintptr) { return T, G } +`) + if prog.NeedsLocalContext() { + t.Fatal("pointer-free zero-value locals enabled a local context") + } + if strings.Contains(ir, `LocalPackage`) || strings.Contains(ir, `EnsureLocalInitializer`) { + t.Fatalf("zero-value direct locals emitted cold-path support:\n%s", ir) + } +} + +func TestExportedFunctionInstallsLocalContext(t *testing.T) { + _, ir := compileLocalitySource(t, `package locality +//llgo:gls +var Pointer *int +//export Exported +func Exported(useLocal bool) *int { + if useLocal { return Pointer } + return nil +} +`) + exported := llvmFunction(t, ir, "Exported") + if got := strings.Count(exported, "EnterLocalContext"); got != 1 { + t.Fatalf("exported function context entries = %d, want 1:\n%s", got, exported) + } + if got := strings.Count(exported, "LeaveLocalContext"); got != 2 { + t.Fatalf("exported function context leaves = %d, want 2:\n%s", got, exported) + } + assertTextOrder(t, exported, + "EnterLocalContext", + "__llgo_local_block", + "LeaveLocalContext", + "ret ptr", + ) +} + +func TestExportedNativeTLSNeedsNoLocalContext(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality +//llgo:tls +var Scalar int +//export Exported +func Exported() int { return Scalar } +`) + if prog.NeedsLocalContext() { + t.Fatal("zero-value native TLS enabled a local context") + } + exported := llvmFunction(t, ir, "Exported") + if strings.Contains(exported, "LocalContext") { + t.Fatalf("native-TLS-only export installed a local context:\n%s", exported) + } +} + +func assertTextOrder(t *testing.T, text string, wants ...string) { + t.Helper() + offset := 0 + for _, want := range wants { + index := strings.Index(text[offset:], want) + if index < 0 { + t.Fatalf("%q not found after offset %d:\n%s", want, offset, text) + } + offset += index + len(want) + } +} + +func TestLocalityLinknameAliasesReuseCanonicalStorage(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality + +//llgo:gls +var Pointer *int +//go:linkname PointerAlias example.com/locality.Pointer +//llgo:gls +var PointerAlias *int + +//llgo:tls +var Scalar int +//go:linkname ScalarAlias example.com/locality.Scalar +//llgo:tls +var ScalarAlias int + +func values() (*int, *int, int, int) { return Pointer, PointerAlias, Scalar, ScalarAlias } +`) + if strings.Contains(ir, "PointerAlias") || strings.Contains(ir, "ScalarAlias") { + t.Fatalf("linkname aliases received independent LLVM storage:\n%s", ir) + } + if got := strings.Count(ir, `@"example.com/locality.Scalar" = thread_local global i64`); got != 1 { + t.Fatalf("canonical scalar globals = %d, want 1:\n%s", got, ir) + } + values := llvmFunction(t, ir, "example.com/locality.values") + if got := strings.Count(values, `call ptr @"example.com/locality.__llgo_local_block"()`); got != 1 { + t.Fatalf("alias package-base calls = %d, want 1:\n%s", got, values) + } + for name, want := range map[string]llssa.LocalStorage{ + "example.com/locality.PointerAlias": llssa.LocalStoragePackage, + "example.com/locality.ScalarAlias": llssa.LocalStorageNativeTLS, + } { + if got, ok := prog.VariableLocality(name); !ok || got.LocalStorage != want { + t.Fatalf("alias metadata %s = %+v, %v; want storage %v", name, got, ok, want) + } + } +} + +func TestLocalityCrossPackageAccessUsesDependencyStorage(t *testing.T) { + fset := token.NewFileSet() + parse := func(name, source string) *ast.File { + file, err := parser.ParseFile(fset, name, source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + return file + } + depFile := parse("dep.go", `package dep +func initialScalar() int { return 1 } +//llgo:tls +var Scalar = initialScalar() +//llgo:gls +var Pointer *int +`) + rootFile := parse("root.go", `package root +import "example.com/dep" +func Values() (int, *int) { return dep.Scalar, dep.Pointer } +`) + check := func(path string, files []*ast.File, imp types.Importer) (*types.Package, *types.Info) { + info := newLocalityTypeInfo() + pkg, err := (&types.Config{Importer: imp}).Check(path, fset, files, info) + if err != nil { + t.Fatal(err) + } + return pkg, info + } + depPkg, depInfo := check("example.com/dep", []*ast.File{depFile}, nil) + rootPkg, rootInfo := check("example.com/root", []*ast.File{rootFile}, importerFunc(func(path string) (*types.Package, error) { + if path == depPkg.Path() { + return depPkg, nil + } + return nil, types.Error{Msg: "unexpected import " + path} + })) + + prog := ssatest.NewProgram(t, nil) + prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) + prog.SetRuntime(localityRuntimePackage()) + for _, input := range []struct { + pkg *types.Package + info *types.Info + files []*ast.File + }{ + {depPkg, depInfo, []*ast.File{depFile}}, + {rootPkg, rootInfo, []*ast.File{rootFile}}, + } { + if err := ParsePkgSyntax(prog, fset, input.pkg, input.files); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, fset, input.pkg, input.info, input.files); err != nil { + t.Fatal(err) + } + } + + goProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions) + depSSA := goProg.CreatePackage(depPkg, []*ast.File{depFile}, depInfo, true) + rootSSA := goProg.CreatePackage(rootPkg, []*ast.File{rootFile}, rootInfo, true) + goProg.Build() + if _, err := NewPackage(prog, depSSA, []*ast.File{depFile}); err != nil { + t.Fatal(err) + } + root, err := NewPackage(prog, rootSSA, []*ast.File{rootFile}) + if err != nil { + t.Fatal(err) + } + ir := root.String() + if !strings.Contains(ir, `@"example.com/dep.Scalar" = external thread_local global i64`) { + t.Fatalf("root package did not reference dependency TLS storage:\n%s", ir) + } + if !strings.Contains(ir, `declare ptr @"example.com/dep.__llgo_local_block"()`) { + t.Fatalf("root package did not reference dependency block accessor:\n%s", ir) + } + if !strings.Contains(ir, `declare void @"example.com/dep.__llgo_tls_init$ensure"()`) { + t.Fatalf("root package did not reference dependency initializer guard:\n%s", ir) + } + if strings.Contains(ir, `define ptr @"example.com/dep.__llgo_local_block"()`) { + t.Fatalf("root package redefined dependency block accessor:\n%s", ir) + } +} + +func TestPrepareRejectsLocalAliasInitializer(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", `package locality +//llgo:tls +var Target int +//go:linkname Alias example.com/locality.Target +//llgo:tls +var Alias = 1 +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + if err != nil { + t.Fatal(err) + } + prog := llssa.NewProgram(nil) + if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, fset, pkg, info, files); err == nil || !strings.Contains(err.Error(), "linkname alias") { + t.Fatalf("PrepareLocalVariables error = %v", err) + } +} + +func TestValidateLocalInitializers(t *testing.T) { + pkg := types.NewPackage("example.com/locality", "locality") + prog := ssatest.NewProgram(t, nil) + name := llssa.FullName(pkg, "value") + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal, HasInitializer: true}) + if err := validateLocalInitializers(prog, pkg); err == nil || !strings.Contains(err.Error(), "inconsistent initializer metadata") { + t.Fatalf("validateLocalInitializers error = %v", err) + } + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal, HasInitializer: true, InitFunc: "example.com/locality.init", InitOrder: 1}) + if err := validateLocalInitializers(prog, pkg); err != nil { + t.Fatal(err) + } +} + +func TestNewPackageReportsLocalityPreparationErrors(t *testing.T) { + tests := []struct { + name string + src string + parseSyntax bool + wantError string + }{ + { + name: "invalid directive", + src: `package locality +//llgo:tls +func invalid() {} +`, + wantError: "applies only to package-level var declarations", + }, + { + name: "unprepared initializer", + src: `package locality +func initialValue() int { return 1 } +//llgo:tls +var value = initialValue() +`, + parseSyntax: true, + wantError: "inconsistent initializer metadata", + }, + { + name: "linkname locality mismatch", + src: `package locality +//llgo:tls +var Target int +//go:linkname Alias example.com/locality.Target +//llgo:gls +var Alias int +`, + wantError: "uses //llgo:gls", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", tt.src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + if err != nil { + t.Fatal(err) + } + goProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions) + ssaPkg := goProg.CreatePackage(pkg, files, info, true) + ssaPkg.Build() + prog := ssatest.NewProgram(t, nil) + if tt.parseSyntax { + if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { + t.Fatal(err) + } + } + if _, err := NewPackage(prog, ssaPkg, files); err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("NewPackage error = %v, want %q", err, tt.wantError) + } + }) + } +} + +func TestPrepareRejectsLocalAliasWithoutLocalTarget(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", `package locality +//go:linkname Value C.value +//llgo:tls +var Value int +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + if err != nil { + t.Fatal(err) + } + prog := ssatest.NewProgram(t, nil) + if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, fset, pkg, info, files); err == nil || !strings.Contains(err.Error(), "is not a local variable") { + t.Fatalf("PrepareLocalVariables error = %v", err) + } +} + +func TestPrepareLocalVariablesEarlyReturns(t *testing.T) { + prog := ssatest.NewProgram(t, nil) + pkg := types.NewPackage("example.com/locality", "locality") + info := &types.Info{} + if err := PrepareLocalVariables(prog, nil, nil, info, nil); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, nil, pkg, nil, nil); err != nil { + t.Fatal(err) + } + if err := PrepareLocalVariables(prog, nil, pkg, info, nil); err != nil { + t.Fatal(err) + } + value := types.NewVar(token.NoPos, pkg, "value", types.Typ[types.Int]) + pkg.Scope().Insert(value) + prog.SetLocalityInfo(llssa.FullName(pkg, value.Name()), llssa.LocalityInfo{Locality: llssa.ThreadLocal, HasInitializer: true}) + info.InitOrder = []*types.Initializer{{Lhs: []*types.Var{value}, Rhs: ast.NewIdent("rhs")}} + if err := PrepareLocalVariables(prog, nil, pkg, info, nil); err == nil || !strings.Contains(err.Error(), "without syntax files") { + t.Fatalf("PrepareLocalVariables without files error = %v", err) + } + (&context{}).initializeLocalGuards(nil) +} + +func TestPrepareLocalVariablesRejectsInvalidMetadata(t *testing.T) { + t.Run("missing object", func(t *testing.T) { + prog := ssatest.NewProgram(t, nil) + pkg := types.NewPackage("example.com/missing", "missing") + prog.SetLocalityInfo(llssa.FullName(pkg, "Value"), llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + if err := PrepareLocalVariables(prog, nil, pkg, &types.Info{}, nil); err == nil || !strings.Contains(err.Error(), "has no variable") { + t.Fatalf("PrepareLocalVariables error = %v", err) + } + }) + t.Run("linkname cycle", func(t *testing.T) { + prog := ssatest.NewProgram(t, nil) + pkg := types.NewPackage("example.com/cycle", "cycle") + first := llssa.FullName(pkg, "First") + second := llssa.FullName(pkg, "Second") + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, "First", types.Typ[types.Int])) + prog.SetLocalityInfo(first, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + prog.SetLinkname(first, second) + prog.SetLinkname(second, first) + if err := PrepareLocalVariables(prog, nil, pkg, &types.Info{}, nil); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + t.Fatalf("PrepareLocalVariables error = %v", err) + } + }) +} + +func TestPlanLocalPackageDiagnostics(t *testing.T) { + prog := ssatest.NewProgram(t, nil) + if plan, err := planLocalPackage(prog, nil); err != nil || len(plan.Variables) != 0 { + t.Fatalf("nil package plan = %+v, %v", plan, err) + } + + pkg := types.NewPackage("example.com/plan", "plan") + ordinary := llssa.FullName(pkg, "Ordinary") + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, "Ordinary", types.Typ[types.Int])) + prog.SetLocalStorage(ordinary, llssa.LocalStorageNativeTLS) + if plan, err := planLocalPackage(prog, pkg); err != nil || len(plan.Variables) != 0 { + t.Fatalf("non-local metadata plan = %+v, %v", plan, err) + } + + missing := llssa.FullName(pkg, "Missing") + prog.SetLocalityInfo(missing, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + if _, err := planLocalPackage(prog, pkg); err == nil || !strings.Contains(err.Error(), "has no variable") { + t.Fatalf("missing object plan error = %v", err) + } +} + +func TestLocalInitializerNameCollision(t *testing.T) { + prog, _ := compileLocalitySource(t, `package locality +func __llgo_local_init_0() {} +//llgo:tls +var value = 1 +`) + info, ok := prog.VariableLocality("example.com/locality.value") + if !ok || !strings.HasSuffix(info.InitFunc, ".__llgo_local_init_1") || info.InitOrder != 1 { + t.Fatalf("value metadata = %+v, %v", info, ok) + } +} + +func TestNamedPointerLocalUsesPackageStorage(t *testing.T) { + prog, ir := compileLocalitySource(t, `package locality +type Handle struct { Pointer *int } +func makeHandle() Handle { return Handle{} } +//llgo:tls +var Value = makeHandle() +func get() Handle { return Value } +`) + info, ok := prog.VariableLocality("example.com/locality.Value") + if !ok || info.LocalStorage != llssa.LocalStoragePackage { + t.Fatalf("named pointer metadata = %+v, %v", info, ok) + } + if !strings.Contains(ir, `call ptr @"example.com/locality.__llgo_local_block"()`) { + t.Fatalf("named pointer did not use package storage:\n%s", ir) + } +} diff --git a/cl/static_init.go b/cl/static_init.go index bbbe94cfb4..fd0f238151 100644 --- a/cl/static_init.go +++ b/cl/static_init.go @@ -85,6 +85,11 @@ func (p *context) collectStaticGlobalInits(pkg *ssa.Package) { if _, rewritten := p.rewriteValue(globalName); rewritten { continue } + if info, ok := p.resolveLocality(llssa.FullName(global.Pkg.Pkg, global.Name())); ok && info.Locality != llssa.LocalityNone { + // Local initializers must remain executable so they can populate the + // current context rather than a process-wide LLVM initializer. + continue + } globals[global] = none{} } } diff --git a/internal/build/build.go b/internal/build/build.go index 7b66fc73e9..14ebc96e1c 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -416,11 +416,27 @@ func Do(args []string, conf *Config) ([]Package, error) { return prog.TypeSizes(sizes) } dedup := packages.NewDeduper() + var syntaxErr error + var syntaxErrMu sync.Mutex + recordSyntaxErr := func(err error) { + syntaxErrMu.Lock() + defer syntaxErrMu.Unlock() + if syntaxErr == nil { + syntaxErr = err + } + } + loadSyntaxErr := func() error { + syntaxErrMu.Lock() + defer syntaxErrMu.Unlock() + return syntaxErr + } dedup.SetPreload(func(pkg *types.Package, files []*ast.File) { if llruntime.SkipToBuild(pkg.Path()) { return } - cl.ParsePkgSyntax(prog, pkg, files) + if err := cl.ParsePkgSyntax(prog, cfg.Fset, pkg, files); err != nil { + recordSyntaxErr(err) + } }) if patterns == nil { @@ -443,6 +459,9 @@ func Do(args []string, conf *Config) ([]Package, error) { if err != nil { return nil, err } + if err := loadSyntaxErr(); err != nil { + return nil, err + } if conf.AllowNoBody { allowMissingFunctionBodies(initial) } @@ -477,6 +496,9 @@ func Do(args []string, conf *Config) ([]Package, error) { if err != nil { return nil, err } + if err := loadSyntaxErr(); err != nil { + return nil, err + } prog.SetRuntime(func() *types.Package { return altPkgs[0].Types @@ -484,7 +506,9 @@ func Do(args []string, conf *Config) ([]Package, error) { prog.SetPython(func() *types.Package { return dedup.Check(llssa.PkgPython).Types }) - preCollectRuntimeLinknames(prog, altPkgs) + if err := prepareLocalVariables(prog, initial, altPkgs); err != nil { + return nil, err + } buildMode := ssaBuildMode cabiOptimize := true @@ -1932,13 +1956,22 @@ func altPkgs(initial []*packages.Package, conf *Config, alts ...string) []string return alts } -func preCollectRuntimeLinknames(prog llssa.Program, pkgs []*packages.Package) { - for _, pkg := range pkgs { - if pkg != nil && pkg.PkgPath == llssa.PkgRuntime && len(pkg.Syntax) != 0 { - cl.PreCollectLinknames(prog, pkg.PkgPath, pkg.Syntax) - return +func prepareLocalVariables(prog llssa.Program, groups ...[]*packages.Package) error { + seen := make(map[*types.Package]bool) + var firstErr error + for _, roots := range groups { + packages.Visit(roots, nil, func(p *packages.Package) { + if firstErr != nil || p.Types == nil || p.IllTyped || seen[p.Types] { + return + } + seen[p.Types] = true + firstErr = cl.PrepareLocalVariables(prog, p.Fset, p.Types, p.TypesInfo, p.Syntax) + }) + if firstErr != nil { + return firstErr } } + return nil } func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, conf *Config, verbose bool) { diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 7f3eb3893c..6584ad3c87 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -11,6 +11,7 @@ import ( gobuild "go/build" "go/parser" "go/token" + "go/types" "io" "os" "os/exec" @@ -21,6 +22,7 @@ import ( "strings" "testing" + "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/buildenv" "github.com/goplus/llgo/internal/crosscompile" "github.com/goplus/llgo/internal/env" @@ -638,7 +640,7 @@ func TestCmpTestNonexistentPatternReturnsError(t *testing.T) { } } -func TestPreCollectRuntimeLinknames(t *testing.T) { +func TestParsePkgSyntaxCollectsRuntimeLinknames(t *testing.T) { prog := llssa.NewProgram(nil) fset := token.NewFileSet() file, err := parser.ParseFile(fset, "runtime.go", `package runtime @@ -649,15 +651,68 @@ func Sigsetjmp() if err != nil { t.Fatalf("ParseFile failed: %v", err) } - preCollectRuntimeLinknames(prog, []*packages.Package{{ - PkgPath: llssa.PkgRuntime, - Syntax: []*ast.File{file}, - }}) + pkg := types.NewPackage(llssa.PkgRuntime, "runtime") + if err := cl.ParsePkgSyntax(prog, fset, pkg, []*ast.File{file}); err != nil { + t.Fatal(err) + } if got, ok := prog.Linkname(llssa.PkgRuntime + ".Sigsetjmp"); !ok || got != "C.sigsetjmp" { t.Fatalf("pre-collected runtime linkname = (%q,%v), want (%q,%v)", got, ok, "C.sigsetjmp", true) } } +func TestPrepareLocalVariables(t *testing.T) { + newLocalPackage := func(path string, withSyntax bool) (*packages.Package, *ast.File) { + pkg := types.NewPackage(path, "local") + value := types.NewVar(token.NoPos, pkg, "value", types.Typ[types.Int]) + pkg.Scope().Insert(value) + info := &types.Info{ + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + InitOrder: []*types.Initializer{{Lhs: []*types.Var{value}, Rhs: ast.NewIdent("rhs")}}, + } + loaded := &packages.Package{Types: pkg, TypesInfo: info} + var file *ast.File + if withSyntax { + file = &ast.File{Name: ast.NewIdent("local")} + loaded.Syntax = []*ast.File{file} + } + return loaded, file + } + + t.Run("filters and deduplicates packages", func(t *testing.T) { + prog := llssa.NewProgram(nil) + loaded, file := newLocalPackage("example.com/local", true) + prog.SetLocalityInfo("example.com/local.value", llssa.LocalityInfo{Locality: llssa.ThreadLocal, HasInitializer: true}) + duplicate := *loaded + + err := prepareLocalVariables(prog, + []*packages.Package{{}, {Types: types.NewPackage("example.com/bad", "bad"), IllTyped: true}, loaded}, + []*packages.Package{&duplicate}, + ) + if err != nil { + t.Fatal(err) + } + if got := len(file.Decls); got != 1 { + t.Fatalf("generated initializer declarations = %d, want 1", got) + } + }) + + t.Run("returns dependency error", func(t *testing.T) { + prog := llssa.NewProgram(nil) + dependency, _ := newLocalPackage("example.com/dependency", false) + prog.SetLocalityInfo("example.com/dependency.value", llssa.LocalityInfo{Locality: llssa.GoroutineLocal, HasInitializer: true}) + root := &packages.Package{ + Types: types.NewPackage("example.com/root", "root"), + Imports: map[string]*packages.Package{"example.com/dependency": dependency}, + } + + err := prepareLocalVariables(prog, []*packages.Package{root}) + if err == nil || !strings.Contains(err.Error(), "without syntax files") { + t.Fatalf("prepareLocalVariables error = %v", err) + } + }) +} + func TestLTOEnabledDefault(t *testing.T) { host := &Config{Target: ""} if host.ltoEnabled() { @@ -1047,6 +1102,70 @@ func F() {} pkgs[0].LPkg.Prog.Dispose() } +func TestDoReportsLocalityDirectiveError(t *testing.T) { + file := filepath.Join(t.TempDir(), "invalid_locality.go") + if err := os.WriteFile(file, []byte(`package invalidlocality + +//llgo:tls +func Invalid() {} +`), 0o644); err != nil { + t.Fatal(err) + } + conf := NewDefaultConf(ModeGen) + if _, err := Do([]string{file}, conf); err == nil || !strings.Contains(err.Error(), "applies only to package-level var declarations") { + t.Fatalf("Do error = %v, want locality directive diagnostic", err) + } +} + +func TestDoReportsLocalityAliasInitializer(t *testing.T) { + file := filepath.Join(t.TempDir(), "invalid_locality_alias.go") + if err := os.WriteFile(file, []byte(`package invalidlocalityalias + +import _ "unsafe" + +//llgo:tls +var Target int + +//go:linkname Alias example.com/target.Value +//llgo:tls +var Alias = 1 +`), 0o644); err != nil { + t.Fatal(err) + } + conf := NewDefaultConf(ModeGen) + if _, err := Do([]string{file}, conf); err == nil || !strings.Contains(err.Error(), "linkname alias") { + t.Fatalf("Do error = %v, want locality alias initializer diagnostic", err) + } +} + +func TestDoReportsAltPackageLocalityDirectiveError(t *testing.T) { + root := t.TempDir() + runtimeDir := filepath.Join(root, "runtime") + runtimePkgDir := filepath.Join(runtimeDir, "internal", "runtime") + if err := os.MkdirAll(runtimePkgDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(runtimeDir, "go.mod"), []byte("module github.com/goplus/llgo/runtime\n\ngo 1.24.0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(runtimePkgDir, "runtime.go"), []byte(`package runtime + +//llgo:gls +func Invalid() {} +`), 0o644); err != nil { + t.Fatal(err) + } + file := filepath.Join(root, "main.go") + if err := os.WriteFile(file, []byte("package main\nfunc main() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("LLGO_ROOT", root) + conf := NewDefaultConf(ModeGen) + if _, err := Do([]string{file}, conf); err == nil || !strings.Contains(err.Error(), "applies only to package-level var declarations") { + t.Fatalf("Do error = %v, want alternate-package locality directive diagnostic", err) + } +} + func TestFormatPackageError(t *testing.T) { tests := []struct { name string diff --git a/internal/build/main_module.go b/internal/build/main_module.go index fa9cdb7afc..c06ee57530 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -246,6 +246,11 @@ func defineEntryFunction(ctx *context, pkg llssa.Package, argcVar, argvVar llssa fnVal.SetUnnamedAddr(true) } b := fn.MakeBody(1) + var localCtx, previousLocalCtx llssa.Expr + hasLocalContext := prog.NeedsLocalContext() + if hasLocalContext { + localCtx, previousLocalCtx = b.EnterLocalContext() + } b.Store(argcVar.Expr, fn.Param(0)) b.Store(argvVar.Expr, fn.Param(1)) if IsStdioNobuf() { @@ -266,6 +271,9 @@ func defineEntryFunction(ctx *context, pkg llssa.Package, argcVar, argvVar llssa if fns.pyFinalize != nil { b.Call(fns.pyFinalize.Expr) } + if hasLocalContext { + b.LeaveLocalContext(localCtx, previousLocalCtx) + } b.Return(prog.IntVal(0, prog.Int32())) return fn } diff --git a/internal/build/main_module_test.go b/internal/build/main_module_test.go index 7793d5fbfa..4e0b16c907 100644 --- a/internal/build/main_module_test.go +++ b/internal/build/main_module_test.go @@ -4,6 +4,8 @@ package build import ( + "go/token" + "go/types" "strings" "testing" @@ -146,6 +148,44 @@ func TestGenMainModuleTestLibraryDefersMainInit(t *testing.T) { } } +func TestGenMainModuleInstallsLocalContextWhenNeeded(t *testing.T) { + llvm.InitializeAllTargets() + t.Setenv(llgoStdioNobuf, "") + prog := llssa.NewProgram(nil) + runtimePkg := types.NewPackage(llssa.PkgRuntime, "runtime") + contextName := types.NewTypeName(token.NoPos, runtimePkg, "LocalContext", nil) + contextType := types.NewNamed(contextName, types.NewStruct(nil, nil), nil) + runtimePkg.Scope().Insert(contextName) + contextPointer := types.NewPointer(contextType) + enterParams := types.NewTuple(types.NewParam(token.NoPos, runtimePkg, "ctx", contextPointer)) + enterResults := types.NewTuple(types.NewParam(token.NoPos, runtimePkg, "previous", types.Typ[types.Uintptr])) + runtimePkg.Scope().Insert(types.NewFunc(token.NoPos, runtimePkg, "EnterLocalContext", types.NewSignatureType(nil, nil, nil, enterParams, enterResults, false))) + leaveParams := types.NewTuple( + types.NewParam(token.NoPos, runtimePkg, "ctx", contextPointer), + types.NewParam(token.NoPos, runtimePkg, "previous", types.Typ[types.Uintptr]), + ) + runtimePkg.Scope().Insert(types.NewFunc(token.NoPos, runtimePkg, "LeaveLocalContext", types.NewSignatureType(nil, nil, nil, leaveParams, nil, false))) + prog.SetRuntime(runtimePkg) + prog.SetLocalityInfo("example.com/state.Value", llssa.LocalityInfo{Locality: llssa.GoroutineLocal}) + prog.SetLocalStorage("example.com/state.Value", llssa.LocalStoragePackage) + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + pkg := &packages.Package{PkgPath: "example.com/foo", ExportFile: "foo.a"} + ir := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{}).LPkg.String() + assertInOrder(t, ir, + "EnterLocalContext", + `call void @"example.com/foo.init"()`, + `call void @"example.com/foo.main"()`, + "LeaveLocalContext", + ) +} + func assertInOrder(t *testing.T, s string, wants ...string) { t.Helper() offset := 0 diff --git a/internal/locality/layout/layout.go b/internal/locality/layout/layout.go new file mode 100644 index 0000000000..0589c39f1f --- /dev/null +++ b/internal/locality/layout/layout.go @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package layout plans the internal storage layout for one package's local variables. It +// is intentionally independent of LLGo SSA, LLVM, the runtime, and the build +// cache so every integration layer consumes the same policy result. +package layout + +import ( + "fmt" + "go/types" + "sort" + + "github.com/goplus/llgo/internal/locality" +) + +// Storage identifies the physical addressing strategy selected for a variable. +type Storage uint8 + +const ( + StorageUnknown Storage = iota + StorageNativeTLS + StoragePackage +) + +// Declaration is the source information needed to plan one variable. +type Declaration struct { + Name string + Type types.Type + Info locality.Info +} + +// Variable is one planned local variable. Field is valid for package storage. +type Variable struct { + Declaration + Storage Storage + Field int +} + +// Initializer identifies one replay helper in Go package initialization order. +type Initializer struct { + Name string + Order int +} + +// Package is a deterministic storage plan. Block contains all pointer-bearing +// TLS and GLS variables in their shared physical package block. +type Package struct { + Path string + Variables []Variable + Block []Variable + Thread []Initializer + Goroutine []Initializer + + byName map[string]int +} + +// Plan creates a deterministic package layout. TLS and GLS remain logically +// distinct, while the current one-thread-per-goroutine backend may share their +// pointer-bearing package block. +func Plan(path string, declarations []Declaration) (Package, error) { + ret := Package{Path: path} + decls := append([]Declaration(nil), declarations...) + sort.Slice(decls, func(i, j int) bool { return decls[i].Name < decls[j].Name }) + ret.byName = make(map[string]int, len(decls)) + initializers := map[locality.Kind]map[int]string{ + locality.Thread: {}, + locality.Goroutine: {}, + } + for _, decl := range decls { + if decl.Info.Locality == locality.None { + continue + } + if decl.Name == "" || decl.Type == nil { + return Package{}, fmt.Errorf("locality layout: incomplete declaration %q", decl.Name) + } + if _, exists := ret.byName[decl.Name]; exists { + return Package{}, fmt.Errorf("locality layout: duplicate declaration %s", decl.Name) + } + if decl.Info.Locality != locality.Thread && decl.Info.Locality != locality.Goroutine { + return Package{}, fmt.Errorf("locality layout: invalid locality for %s", decl.Name) + } + prepared := decl.Info.InitFunc != "" && decl.Info.InitOrder != 0 + if decl.Info.HasInitializer != prepared { + return Package{}, fmt.Errorf("locality layout: inconsistent initializer metadata for %s", decl.Name) + } + variable := Variable{Declaration: decl, Field: -1, Storage: StorageForType(decl.Type)} + if variable.Storage == StoragePackage { + variable.Field = len(ret.Block) + ret.Block = append(ret.Block, variable) + } + ret.byName[decl.Name] = len(ret.Variables) + ret.Variables = append(ret.Variables, variable) + if decl.Info.InitFunc != "" { + byOrder := initializers[decl.Info.Locality] + if current, exists := byOrder[decl.Info.InitOrder]; exists && current != decl.Info.InitFunc { + return Package{}, fmt.Errorf("locality layout: initializer order %d names both %s and %s", decl.Info.InitOrder, current, decl.Info.InitFunc) + } + byOrder[decl.Info.InitOrder] = decl.Info.InitFunc + } + } + ret.Thread = orderedInitializers(initializers[locality.Thread]) + ret.Goroutine = orderedInitializers(initializers[locality.Goroutine]) + return ret, nil +} + +// StorageForType returns the physical storage class for a local variable type. +// Pointer-free values use native LLVM TLS; values visible to the GC share the +// package block rooted by LocalContext. +func StorageForType(typ types.Type) Storage { + if hasPointers(typ) { + return StoragePackage + } + return StorageNativeTLS +} + +func hasPointers(typ types.Type) bool { + typ = types.Unalias(typ) + switch typ := typ.(type) { + case *types.Basic: + return typ.Kind() == types.String || typ.Kind() == types.UnsafePointer + case *types.Pointer, *types.Slice, *types.Map, *types.Chan, *types.Signature, *types.Interface: + return true + case *types.Array: + return typ.Len() != 0 && hasPointers(typ.Elem()) + case *types.Struct: + for i := 0; i < typ.NumFields(); i++ { + if hasPointers(typ.Field(i).Type()) { + return true + } + } + return false + case *types.Named: + return hasPointers(typ.Underlying()) + case *types.TypeParam: + return true + default: + return false + } +} + +func orderedInitializers(byOrder map[int]string) []Initializer { + ret := make([]Initializer, 0, len(byOrder)) + for order, name := range byOrder { + ret = append(ret, Initializer{Name: name, Order: order}) + } + sort.Slice(ret, func(i, j int) bool { return ret[i].Order < ret[j].Order }) + return ret +} + +// Lookup returns a variable from the package plan. +func (p Package) Lookup(name string) (Variable, bool) { + index, ok := p.byName[name] + if !ok { + return Variable{}, false + } + return p.Variables[index], true +} + +// Initializers returns the ordered replay helpers for kind. +func (p Package) Initializers(kind locality.Kind) []Initializer { + if kind == locality.Thread { + return p.Thread + } + if kind == locality.Goroutine { + return p.Goroutine + } + return nil +} + +// BlockName returns the shared package-block accessor symbol. +func BlockName(path string) string { return qualify(path, "__llgo_local_block") } + +// BlockKeyName returns the shared package-block descriptor symbol. +func BlockKeyName(path string) string { return qualify(path, "__llgo_local_key") } + +// InitName returns the package/kind initializer dispatcher symbol. +func InitName(path string, kind locality.Kind) string { + return qualify(path, "__llgo_"+kind.String()+"_init") +} + +// EnsureName returns the package/kind first-use initializer symbol. +func EnsureName(path string, kind locality.Kind) string { return InitName(path, kind) + "$ensure" } + +// GuardName returns the package/kind native TLS state symbol. +func GuardName(path string, kind locality.Kind) string { return InitName(path, kind) + "$guard" } + +// FailureKeyName returns the package/kind initializer failure key symbol. +func FailureKeyName(path string, kind locality.Kind) string { return InitName(path, kind) + "$failure" } + +func qualify(path, name string) string { + if path == "" { + return name + } + return path + "." + name +} diff --git a/internal/locality/layout/layout_test.go b/internal/locality/layout/layout_test.go new file mode 100644 index 0000000000..3aabeb2b79 --- /dev/null +++ b/internal/locality/layout/layout_test.go @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package layout + +import ( + "go/token" + "go/types" + "strings" + "testing" + + "github.com/goplus/llgo/internal/locality" +) + +func TestPlanSharesPointerStorageAndPreservesKinds(t *testing.T) { + plan, err := Plan("example.com/state", []Declaration{ + {Name: "example.com/state.Ignored", Type: types.Typ[types.Int]}, + {Name: "example.com/state.Z", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Goroutine}}, + {Name: "example.com/state.P", Type: types.NewPointer(types.Typ[types.Int]), Info: locality.Info{Locality: locality.Thread, HasInitializer: true, InitFunc: "p.init1", InitOrder: 2}}, + {Name: "example.com/state.A", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Goroutine, HasInitializer: true, InitFunc: "p.init0", InitOrder: 1}}, + {Name: "example.com/state.Q", Type: types.NewSlice(types.Typ[types.Byte]), Info: locality.Info{Locality: locality.Goroutine}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plan.Variables) != 4 || len(plan.Block) != 2 { + t.Fatalf("plan sizes = %d/%d", len(plan.Variables), len(plan.Block)) + } + if got, _ := plan.Lookup("example.com/state.Z"); got.Storage != StorageNativeTLS || got.Info.Locality != locality.Goroutine { + t.Fatalf("scalar GLS plan = %+v", got) + } + if got, _ := plan.Lookup("example.com/state.P"); got.Storage != StoragePackage || got.Field != 0 { + t.Fatalf("pointer TLS plan = %+v", got) + } + if got, _ := plan.Lookup("example.com/state.Q"); got.Storage != StoragePackage || got.Field != 1 { + t.Fatalf("slice GLS plan = %+v", got) + } + if len(plan.Thread) != 1 || plan.Thread[0].Name != "p.init1" { + t.Fatalf("thread initializers = %+v", plan.Thread) + } + if len(plan.Goroutine) != 1 || plan.Goroutine[0].Name != "p.init0" { + t.Fatalf("goroutine initializers = %+v", plan.Goroutine) + } + if got := plan.Initializers(locality.Thread); len(got) != 1 || got[0].Name != "p.init1" { + t.Fatalf("Initializers(thread) = %+v", got) + } + if got := plan.Initializers(locality.Goroutine); len(got) != 1 || got[0].Name != "p.init0" { + t.Fatalf("Initializers(goroutine) = %+v", got) + } + if got := plan.Initializers(locality.None); got != nil { + t.Fatalf("Initializers(none) = %+v", got) + } + if _, ok := plan.Lookup("example.com/state.Missing"); ok { + t.Fatal("Lookup found a missing variable") + } +} + +func TestOrderedInitializers(t *testing.T) { + got := orderedInitializers(map[int]string{3: "p.third", 1: "p.first", 2: "p.second"}) + if len(got) != 3 || got[0].Name != "p.first" || got[1].Name != "p.second" || got[2].Name != "p.third" { + t.Fatalf("ordered initializers = %+v", got) + } +} + +func TestPlanRejectsInvalidDeclarations(t *testing.T) { + tests := []struct { + name string + in []Declaration + want string + }{ + {"missing type", []Declaration{{Name: "p.x", Info: locality.Info{Locality: locality.Thread}}}, "incomplete"}, + {"duplicate", []Declaration{{Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread}}, {Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread}}}, "duplicate"}, + {"invalid kind", []Declaration{{Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Kind(99)}}}, "invalid locality"}, + {"unprepared", []Declaration{{Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread, HasInitializer: true}}}, "inconsistent initializer metadata"}, + {"unexpected helper", []Declaration{{Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread, InitFunc: "p.a", InitOrder: 1}}}, "inconsistent initializer metadata"}, + {"order conflict", []Declaration{{Name: "p.x", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread, HasInitializer: true, InitFunc: "p.a", InitOrder: 1}}, {Name: "p.y", Type: types.Typ[types.Int], Info: locality.Info{Locality: locality.Thread, HasInitializer: true, InitFunc: "p.b", InitOrder: 1}}}, "names both"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := Plan("p", test.in); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Plan error = %v, want %q", err, test.want) + } + }) + } +} + +func TestNames(t *testing.T) { + if got := BlockName("example.com/p"); got != "example.com/p.__llgo_local_block" { + t.Fatal(got) + } + if got := BlockKeyName("example.com/p"); got != "example.com/p.__llgo_local_key" { + t.Fatal(got) + } + if got := InitName("example.com/p", locality.Thread); got != "example.com/p.__llgo_tls_init" { + t.Fatal(got) + } + if got := EnsureName("example.com/p", locality.Goroutine); got != "example.com/p.__llgo_gls_init$ensure" { + t.Fatal(got) + } + if got := GuardName("", locality.Thread); got != "__llgo_tls_init$guard" { + t.Fatal(got) + } + if got := FailureKeyName("", locality.Goroutine); got != "__llgo_gls_init$failure" { + t.Fatal(got) + } +} + +func TestStorageForType(t *testing.T) { + if got := StorageForType(types.Typ[types.Uintptr]); got != StorageNativeTLS { + t.Fatalf("uintptr storage = %v", got) + } + if got := StorageForType(types.NewArray(types.NewPointer(types.Typ[types.Int]), 0)); got != StorageNativeTLS { + t.Fatalf("zero-length pointer array storage = %v", got) + } + if got := StorageForType(types.NewStruct([]*types.Var{ + types.NewField(0, nil, "value", types.NewPointer(types.Typ[types.Int]), false), + }, nil)); got != StoragePackage { + t.Fatalf("pointer struct storage = %v", got) + } +} + +func TestHasPointers(t *testing.T) { + pkg := types.NewPackage("example.com/types", "types") + namedInt := types.NewNamed(types.NewTypeName(token.NoPos, pkg, "Int", nil), types.Typ[types.Int], nil) + typeParam := types.NewTypeParam(types.NewTypeName(token.NoPos, pkg, "T", nil), types.NewInterfaceType(nil, nil).Complete()) + tests := []struct { + typ types.Type + want bool + }{ + {types.Typ[types.Int], false}, + {types.Typ[types.String], true}, + {types.Typ[types.UnsafePointer], true}, + {types.NewPointer(types.Typ[types.Int]), true}, + {types.NewSlice(types.Typ[types.Int]), true}, + {types.NewMap(types.Typ[types.Int], types.Typ[types.Int]), true}, + {types.NewChan(types.SendRecv, types.Typ[types.Int]), true}, + {types.NewSignatureType(nil, nil, nil, nil, nil, false), true}, + {types.NewInterfaceType(nil, nil).Complete(), true}, + {types.NewArray(types.Typ[types.Int], 1), false}, + {types.NewArray(types.NewPointer(types.Typ[types.Int]), 0), false}, + {types.NewArray(types.NewPointer(types.Typ[types.Int]), 1), true}, + {types.NewStruct([]*types.Var{types.NewVar(token.NoPos, pkg, "n", types.Typ[types.Int])}, nil), false}, + {types.NewStruct([]*types.Var{types.NewVar(token.NoPos, pkg, "p", types.NewPointer(types.Typ[types.Int]))}, nil), true}, + {namedInt, false}, + {typeParam, true}, + {types.NewTuple(), false}, + } + for _, test := range tests { + if got := hasPointers(test.typ); got != test.want { + t.Fatalf("hasPointers(%v) = %v, want %v", test.typ, got, test.want) + } + } +} diff --git a/ssa/decl.go b/ssa/decl.go index df70a66a0e..a575dd1bee 100644 --- a/ssa/decl.go +++ b/ssa/decl.go @@ -117,9 +117,25 @@ func (p Package) NewVarEx(name string, t Type) Global { return p.doNewVar(name, t) } +// NewThreadLocalVar creates a native TLS variable. Unlike NewVar, it keeps +// independent storage for zero-sized values instead of using the module-wide +// zero-sized allocation sentinel. +func (p Package) NewThreadLocalVar(name string, typ types.Type, bg Background) Global { + if v, ok := p.vars[name]; ok { + v.impl.SetThreadLocal(true) + return v + } + t := p.Prog.Type(typ, bg) + return p.doNewVarEx(name, t, true) +} + func (p Package) doNewVar(name string, t Type) Global { + return p.doNewVarEx(name, t, false) +} + +func (p Package) doNewVarEx(name string, t Type, threadLocal bool) Global { typ := p.Prog.Elem(t).ll - if p.Prog.td.TypeAllocSize(typ) == 0 { + if !threadLocal && p.Prog.td.TypeAllocSize(typ) == 0 { var rt *types.Package if p.Prog.rt != nil || p.Prog.rtget != nil { rt = p.Prog.runtime() @@ -141,6 +157,7 @@ func (p Package) doNewVar(name string, t Type) Global { } } gbl := llvm.AddGlobal(p.mod, typ, name) + gbl.SetThreadLocal(threadLocal) alignment := p.Prog.td.ABITypeAlignment(typ) gbl.SetAlignment(alignment) ret := &aGlobal{Expr{gbl, t}} diff --git a/ssa/goroutine.go b/ssa/goroutine.go index 278f4e7ec8..8460d69ca0 100644 --- a/ssa/goroutine.go +++ b/ssa/goroutine.go @@ -85,6 +85,11 @@ func (p Package) routine(t Type, fn Expr, buildCall func(Builder, Expr, ...Expr) prog := p.Prog routine := p.NewFunc(p.routineName(), prog.tyRoutine(), InC) b := routine.MakeBody(1) + var localCtx, previousLocalCtx Expr + hasLocalContext := prog.NeedsLocalContext() + if hasLocalContext { + localCtx, previousLocalCtx = b.EnterLocalContext() + } param := routine.Param(0) data := Expr{llvm.CreateLoad(b.impl, t.ll, param.impl), t} args := make([]Expr, n) @@ -100,6 +105,9 @@ func (p Package) routine(t Type, fn Expr, buildCall func(Builder, Expr, ...Expr) buildCall(b, fn, args...) lastInst := b.impl.GetInsertBlock().LastInstruction() if lastInst.IsNil() || lastInst.IsAUnreachableInst().IsNil() { + if hasLocalContext { + b.LeaveLocalContext(localCtx, previousLocalCtx) + } b.Return(prog.Nil(prog.VoidPtr())) } return routine.Expr diff --git a/ssa/goroutine_patch_test.go b/ssa/goroutine_patch_test.go index b6b245cd45..53d704dd42 100644 --- a/ssa/goroutine_patch_test.go +++ b/ssa/goroutine_patch_test.go @@ -53,6 +53,29 @@ func TestGoClosureStartupUsesGCManagedMemory(t *testing.T) { if got := strings.Count(ir, `"github.com/goplus/llgo/runtime/internal/runtime.AllocU"`); got < 1 { t.Fatalf("expected closure ctx to use AllocU, got %d:\n%s", got, ir) } + if strings.Contains(ir, "EnterLocalContext") { + t.Fatalf("program without context-backed locals paid locality entry cost:\n%s", ir) + } +} + +func TestGoInstallsContextForContextBackedLocals(t *testing.T) { + prog := ssatest.NewProgram(t, nil) + prog.SetLocalityInfo("example.com/state.Value", ssa.LocalityInfo{Locality: ssa.GoroutineLocal}) + prog.SetLocalStorage("example.com/state.Value", ssa.LocalStoragePackage) + pkg := prog.NewPackage("bar", "foo/bar") + outer := pkg.NewFunc("outer", ssa.NoArgsNoRet, ssa.InGo) + b := outer.MakeBody(1) + b.Go(ssa.Nil, func(b ssa.Builder, _ ssa.Expr, args ...ssa.Expr) ssa.Expr { + return ssa.Expr{} + }) + b.Return() + + ir := pkg.String() + for _, want := range []string{"LocalContext", "EnterLocalContext", "LeaveLocalContext"} { + if !strings.Contains(ir, want) { + t.Fatalf("goroutine wrapper missing %q:\n%s", want, ir) + } + } } func TestGoPanicRoutineDoesNotReturnAfterUnreachable(t *testing.T) { diff --git a/ssa/local_context.go b/ssa/local_context.go new file mode 100644 index 0000000000..9739ff19cc --- /dev/null +++ b/ssa/local_context.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import "go/types" + +// EnterLocalContext creates the stack root used by TLS/GLS locality blocks and +// installs it for the current outermost Go entry. previous is nonzero only for +// a nested entry that inherited an existing context. +func (b Builder) EnterLocalContext() (ctx, previous Expr) { + fn := b.Pkg.rtFunc("EnterLocalContext") + params := fn.raw.Type.(*types.Signature).Params() + ctxPtr := b.Prog.rawType(params.At(0).Type()) + ctxType := b.Prog.rawType(ctxPtr.RawType().(*types.Pointer).Elem()) + ctx = b.Alloc(ctxType, false) + previous = b.Call(fn, ctx) + return +} + +// LeaveLocalContext restores an inherited context or drops the stack roots +// installed by EnterLocalContext. +func (b Builder) LeaveLocalContext(ctx, previous Expr) { + b.Call(b.Pkg.rtFunc("LeaveLocalContext"), ctx, previous) +} diff --git a/ssa/locality.go b/ssa/locality.go new file mode 100644 index 0000000000..4cfbb68fa5 --- /dev/null +++ b/ssa/locality.go @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "fmt" + "go/types" + "sort" + "strings" + "sync" + + "github.com/goplus/llgo/internal/locality" + localitylayout "github.com/goplus/llgo/internal/locality/layout" +) + +type Locality = locality.Kind + +const ( + LocalityNone = locality.None + ThreadLocal = locality.Thread + GoroutineLocal = locality.Goroutine +) + +type LocalityInfo = locality.Info +type LocalStorage = localitylayout.Storage + +const ( + LocalStorageUnknown = localitylayout.StorageUnknown + LocalStorageNativeTLS = localitylayout.StorageNativeTLS + LocalStoragePackage = localitylayout.StoragePackage +) + +// VariableLocality is the locality metadata attached to one package variable. +type VariableLocality struct { + LocalStorage LocalStorage + locality.Info +} + +type localityInfos struct { + mu sync.RWMutex + entries map[string]VariableLocality + parsedPackages map[*types.Package]struct{} +} + +func newLocalityInfos() *localityInfos { + return &localityInfos{ + entries: make(map[string]VariableLocality), + parsedPackages: make(map[*types.Package]struct{}), + } +} + +func (p *localityInfos) update(name string, update func(*VariableLocality)) { + p.mu.Lock() + info := p.entries[name] + update(&info) + p.entries[name] = info + p.mu.Unlock() +} + +func (p Program) SetLocalityInfo(name string, info LocalityInfo) { + p.localities.update(name, func(current *VariableLocality) { current.Info = info }) +} + +func (p Program) SetLocalStorage(name string, storage LocalStorage) { + p.localities.update(name, func(info *VariableLocality) { info.LocalStorage = storage }) +} + +func (p Program) VariableLocality(name string) (VariableLocality, bool) { + p.localities.mu.RLock() + info, ok := p.localities.entries[name] + p.localities.mu.RUnlock() + return info, ok +} + +// ResolveLocality follows linkname aliases and returns the canonical declaration +// name together with its merged locality metadata. +func (p Program) ResolveLocality(name string) (string, VariableLocality, bool, error) { + lookup := func(name string) (VariableLocality, bool) { + p.localities.mu.RLock() + info, ok := p.localities.entries[name] + p.localities.mu.RUnlock() + return info, ok + } + return resolveLocality(lookup, p.Linkname, name) +} + +func resolveLocality(lookup func(string) (VariableLocality, bool), linkname func(string) (string, bool), name string) (string, VariableLocality, bool, error) { + result, ok := lookup(name) + if !ok { + result = VariableLocality{} + } + seen := make(map[string]bool) + current := name + for { + if seen[current] { + return "", VariableLocality{}, false, fmt.Errorf("declaration linkname cycle involving %s", current) + } + seen[current] = true + target, hasLink := linkname(current) + target = strings.TrimPrefix(target, "go:") + if !hasLink || target == "" || target == current { + return current, result, ok, nil + } + targetInfo, exists := lookup(target) + if exists && targetInfo.Locality != locality.None { + switch { + case result.Locality == locality.None: + result = targetInfo + ok = true + case result.Locality != targetInfo.Locality: + return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s uses %s but target %s uses %s", name, locality.Directive(result.Locality), target, locality.Directive(targetInfo.Locality)) + case hasInitialization(result.Info) && hasInitialization(targetInfo.Info) && result.Info != targetInfo.Info: + return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s and target %s have incompatible local initializers", name, target) + case !hasInitialization(result.Info): + result.Info = targetInfo.Info + } + if result.LocalStorage == LocalStorageUnknown { + result.LocalStorage = targetInfo.LocalStorage + } else if targetInfo.LocalStorage != LocalStorageUnknown && result.LocalStorage != targetInfo.LocalStorage { + return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s and target %s have incompatible local storage", name, target) + } + } + current = target + } +} + +func hasInitialization(info locality.Info) bool { + return info.HasInitializer || info.InitFunc != "" || info.InitOrder != 0 +} + +func (p Program) ValidateLocalities(pkgPath string) error { + prefix := pkgPath + "." + p.localities.mu.RLock() + names := make([]string, 0) + for name := range p.localities.entries { + if strings.HasPrefix(name, prefix) { + names = append(names, name) + } + } + p.localities.mu.RUnlock() + sort.Strings(names) + for _, name := range names { + if _, _, _, err := p.ResolveLocality(name); err != nil { + return err + } + } + return nil +} + +func (p Program) PackageSyntaxParsed(pkg *types.Package) bool { + p.localities.mu.RLock() + _, ok := p.localities.parsedPackages[pkg] + p.localities.mu.RUnlock() + return ok +} + +func (p Program) MarkPackageSyntaxParsed(pkg *types.Package) { + p.localities.mu.Lock() + p.localities.parsedPackages[pkg] = struct{}{} + p.localities.mu.Unlock() +} + +func (p Program) PackageLocalities(pkgPath string) map[string]VariableLocality { + prefix := pkgPath + "." + ret := make(map[string]VariableLocality) + p.localities.mu.RLock() + for name, info := range p.localities.entries { + if info.Locality != locality.None && strings.HasPrefix(name, prefix) { + ret[name] = info + } + } + p.localities.mu.RUnlock() + return ret +} + +func (p Program) NeedsLocalContext() bool { + p.localities.mu.RLock() + defer p.localities.mu.RUnlock() + for _, info := range p.localities.entries { + if info.Locality != locality.None && (info.LocalStorage != LocalStorageNativeTLS || hasInitialization(info.Info)) { + return true + } + } + return false +} diff --git a/ssa/locality_test.go b/ssa/locality_test.go new file mode 100644 index 0000000000..991c80808d --- /dev/null +++ b/ssa/locality_test.go @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "go/types" + "strings" + "testing" +) + +func TestLocalityInfos(t *testing.T) { + prog := NewProgram(nil) + pkg := types.NewPackage("example.com/p", "p") + if prog.PackageSyntaxParsed(pkg) { + t.Fatal("new package was already marked as parsed") + } + prog.MarkPackageSyntaxParsed(pkg) + if !prog.PackageSyntaxParsed(pkg) { + t.Fatal("package syntax parsed marker was not retained") + } + if prog.PackageSyntaxParsed(types.NewPackage("example.com/p", "p")) { + t.Fatal("syntax marker was shared by distinct package objects") + } + + name := "example.com/p.value" + prog.SetLocalityInfo(name, LocalityInfo{ + Locality: GoroutineLocal, + HasInitializer: true, + InitFunc: "example.com/p.initLocal", + InitOrder: 1, + }) + prog.SetLocalStorage(name, LocalStoragePackage) + + want, ok := prog.VariableLocality(name) + if !ok || want.Locality != GoroutineLocal || want.LocalStorage != LocalStoragePackage || !want.HasInitializer || want.InitFunc == "" || want.InitOrder != 1 { + t.Fatalf("VariableLocality(%q) = %+v, %v", name, want, ok) + } + + prog.SetLocalStorage("example.com/p.ordinary", LocalStorageNativeTLS) + decls := prog.PackageLocalities("example.com/p") + if len(decls) != 1 || decls[name] != want { + t.Fatalf("PackageLocalities = %+v", decls) + } + delete(decls, name) + if _, ok := prog.VariableLocality(name); !ok { + t.Fatal("mutating PackageLocalities changed program metadata") + } +} + +func TestNeedsLocalContext(t *testing.T) { + prog := NewProgram(nil) + if prog.NeedsLocalContext() { + t.Fatal("empty program needs a local context") + } + name := "example.com/p.value" + prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal}) + if !prog.NeedsLocalContext() { + t.Fatal("unknown local storage did not conservatively require a context") + } + prog.SetLocalStorage(name, LocalStorageNativeTLS) + if prog.NeedsLocalContext() { + t.Fatal("native TLS required a local context") + } + prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal, HasInitializer: true, InitFunc: "example.com/p.initValue", InitOrder: 1}) + if !prog.NeedsLocalContext() { + t.Fatal("native TLS initializer failure storage did not require a context") + } + prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal}) + prog.SetLocalStorage(name, LocalStoragePackage) + if !prog.NeedsLocalContext() { + t.Fatal("context storage was not detected") + } +} + +func TestResolveLinknameLocality(t *testing.T) { + prog := NewProgram(nil) + target := "example.com/target.Value" + alias := "example.com/alias.Value" + prog.SetLocalityInfo(target, LocalityInfo{Locality: ThreadLocal, HasInitializer: true, InitFunc: "example.com/target.initValue", InitOrder: 1}) + prog.SetLocalStorage(target, LocalStoragePackage) + prog.SetLinkname(alias, target) + + _, got, ok, err := prog.ResolveLocality(alias) + if err != nil { + t.Fatal(err) + } + if !ok || got.Locality != ThreadLocal || got.LocalStorage != LocalStoragePackage || got.InitFunc != "example.com/target.initValue" || got.InitOrder != 1 { + t.Fatalf("ResolveLocality(%q) = %+v, %v", alias, got, ok) + } + if err := prog.ValidateLocalities("example.com/alias"); err != nil { + t.Fatal(err) + } + sameKind := "example.com/alias.SameKind" + prog.SetLinkname(sameKind, target) + prog.SetLocalityInfo(sameKind, LocalityInfo{Locality: ThreadLocal}) + if _, got, ok, err := prog.ResolveLocality(sameKind); err != nil || !ok || got.InitFunc != "example.com/target.initValue" { + t.Fatalf("same-kind ResolveLocality(%q) = %+v, %v", sameKind, got, ok) + } + + incompatible := "example.com/alias.Incompatible" + prog.SetLinkname(incompatible, target) + prog.SetLocalityInfo(incompatible, LocalityInfo{Locality: ThreadLocal, HasInitializer: true, InitFunc: "example.com/alias.initValue", InitOrder: 1}) + if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "incompatible local initializers") { + t.Fatalf("initializer mismatch error = %v", err) + } + targetDecl, _ := prog.VariableLocality(target) + prog.SetLocalityInfo(incompatible, targetDecl.Info) + + storageMismatch := "example.com/alias.StorageMismatch" + prog.SetLinkname(storageMismatch, target) + prog.SetLocalityInfo(storageMismatch, LocalityInfo{Locality: ThreadLocal}) + prog.SetLocalStorage(storageMismatch, LocalStorageNativeTLS) + if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "incompatible local storage") { + t.Fatalf("storage mismatch error = %v", err) + } + prog.SetLocalStorage(storageMismatch, LocalStoragePackage) + + prog.SetLocalityInfo(alias, LocalityInfo{Locality: GoroutineLocal}) + if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "uses //llgo:gls") { + t.Fatalf("locality mismatch error = %v", err) + } +} + +func TestValidateLocalityLinknameCycle(t *testing.T) { + prog := NewProgram(nil) + prog.SetLinkname("example.com/p.First", "example.com/p.Second") + prog.SetLinkname("example.com/p.Second", "example.com/p.First") + prog.SetLocalityInfo("example.com/p.First", LocalityInfo{Locality: ThreadLocal}) + if err := prog.ValidateLocalities("example.com/p"); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + t.Fatalf("linkname cycle error = %v", err) + } +} + +func TestValidateLocalityAllowsSelfLinkname(t *testing.T) { + prog := NewProgram(nil) + name := "example.com/p.Value" + prog.SetLinkname(name, name) + prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal}) + if err := prog.ValidateLocalities("example.com/p"); err != nil { + t.Fatal(err) + } + if _, got, ok, err := prog.ResolveLocality(name); err != nil || !ok || got.Locality != ThreadLocal { + t.Fatalf("ResolveLocality(%q) = %+v, %v", name, got, ok) + } +} diff --git a/ssa/package.go b/ssa/package.go index 71d214f733..64263ea9fc 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -23,6 +23,7 @@ import ( "log" "runtime" "strconv" + "sync" "unsafe" "github.com/goplus/llgo/internal/env" @@ -225,7 +226,9 @@ type aProgram struct { printfTy *types.Signature paramObjPtr_ *types.Var - linkname map[string]string // pkgPath.nameInPkg => linkname + linknameMu sync.RWMutex + linkname map[string]string // pkgPath.nameInPkg => linkname + localities *localityInfos noInterface map[string]none // pkgPath.T.method or pkgPath.(*T).method abiSymbol map[string]*AbiSymbol // abi symbol name => AbiSymbol @@ -316,7 +319,8 @@ func NewProgram(target *Target) Program { ctx: ctx, gocvt: newGoTypes(), target: target, td: td, tm: tm, is32Bits: is32Bits, ptrSize: td.PointerSize(), named: make(map[string]Type), fnnamed: make(map[string]int), - linkname: make(map[string]string), noInterface: make(map[string]none), abiSymbol: make(map[string]*AbiSymbol), + linkname: make(map[string]string), localities: newLocalityInfos(), + noInterface: make(map[string]none), abiSymbol: make(map[string]*AbiSymbol), debugInfoOptimized: target.effectiveOptLevel() != optlevel.O0, } prog.abi.Init(uintptr(prog.ptrSize), (*goProgram)(unsafe.Pointer(prog))) @@ -413,11 +417,15 @@ func (p Program) SetTypeBackground(fullName string, bg Background) { } func (p Program) SetLinkname(name, link string) { + p.linknameMu.Lock() p.linkname[name] = link + p.linknameMu.Unlock() } func (p Program) Linkname(name string) (link string, ok bool) { + p.linknameMu.RLock() link, ok = p.linkname[name] + p.linknameMu.RUnlock() return } @@ -896,6 +904,11 @@ func (p Package) rtFunc(fnName string) Expr { return p.NewFunc(name, sig, InGo).Expr } +// RuntimeFunc returns a declaration for a function in LLGo's internal runtime. +func (p Package) RuntimeFunc(fnName string) Expr { + return p.rtFunc(fnName) +} + func (p Package) cFunc(fullName string, sig *types.Signature) Expr { return p.NewFunc(fullName, sig, InC).Expr } diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 15bca0945e..38085a627b 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -2114,6 +2114,25 @@ source_filename = "foo/bar" `) } +func TestThreadLocalVar(t *testing.T) { + prog := NewProgram(nil) + pkg := prog.NewPackage("bar", "foo/bar") + a := pkg.NewThreadLocalVar("a", types.NewPointer(types.Typ[types.Int]), InGo) + if got := pkg.NewThreadLocalVar("a", types.NewPointer(types.Typ[types.Int]), InGo); got != a { + t.Fatal("NewThreadLocalVar(a) did not reuse the existing global") + } + a.InitNil() + empty := types.NewStruct(nil, nil) + z := pkg.NewThreadLocalVar("z", types.NewPointer(empty), InGo) + z.InitNil() + assertPkg(t, pkg, `; ModuleID = 'foo/bar' +source_filename = "foo/bar" + +@a = thread_local global i64 0, align 8 +@z = thread_local global {} zeroinitializer, align 1 +`) +} + func TestConst(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("bar", "foo/bar") @@ -2893,7 +2912,7 @@ func TestRtFuncResolvesLinkname(t *testing.T) { return name }) - if got := pkg.rtFunc("Sigsetjmp").impl.Name(); got != "sigsetjmp" { + if got := pkg.RuntimeFunc("Sigsetjmp").impl.Name(); got != "sigsetjmp" { t.Fatalf("rtFunc linkname = %q, want %q", got, "sigsetjmp") } } diff --git a/ssa/type.go b/ssa/type.go index 2b343dba32..98c9fc4848 100644 --- a/ssa/type.go +++ b/ssa/type.go @@ -194,6 +194,11 @@ func (p Program) SizeOf(typ Type, n ...int64) uint64 { return size } +// AlignOf returns the ABI alignment of typ for the current target. +func (p Program) AlignOf(typ Type) uint64 { + return uint64(p.td.ABITypeAlignment(typ.ll)) +} + // OffsetOf returns the offset of a field in a struct. func (p Program) OffsetOf(typ Type, i int) uint64 { return p.td.ElementOffset(typ.ll, i) From fe48ceb8fefefd6dd562385e7c7091bfaaf079e0 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 14 Jul 2026 19:09:13 +0800 Subject: [PATCH 04/13] build: key local context mode in package cache --- internal/build/collect.go | 1 + internal/build/collect_test.go | 73 ++++++++++++++++++++++++++++++++++ internal/build/fingerprint.go | 5 ++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/internal/build/collect.go b/internal/build/collect.go index e89db4d1c5..474a0cec46 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -114,6 +114,7 @@ func (c *context) collectCommonInputs(m *manifestBuilder) { m.common.EmitDWARF = shouldEmitDebugInfo(c.buildConf, &c.crossCompile) m.common.PCLNMode = effectivePCLNMode(c.buildConf).String() m.common.DisableBoundsChecks = c.buildConf.DisableBoundsChecks + m.common.LocalContext = c.prog != nil && c.prog.NeedsLocalContext() // Compiler configuration if c.crossCompile.CC != "" { diff --git a/internal/build/collect_test.go b/internal/build/collect_test.go index 0dd9d7ca4d..d6a8c585d5 100644 --- a/internal/build/collect_test.go +++ b/internal/build/collect_test.go @@ -31,6 +31,7 @@ import ( "github.com/goplus/llgo/internal/lto" "github.com/goplus/llgo/internal/meta" "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" gopackages "golang.org/x/tools/go/packages" ) @@ -360,6 +361,78 @@ func TestCollectFingerprintCanonicalizesPCLNEnvironment(t *testing.T) { } } +func TestCollectFingerprintLocalContextMode(t *testing.T) { + td := t.TempDir() + goFile := filepath.Join(td, "state.go") + if err := os.WriteFile(goFile, []byte("package state"), 0644); err != nil { + t.Fatal(err) + } + newPackage := func() *aPackage { + return &aPackage{Package: &packages.Package{ + ID: "example.com/state", + PkgPath: "example.com/state", + GoFiles: []string{goFile}, + }} + } + newContext := func(prog llssa.Program) *context { + return &context{ + conf: &packages.Config{}, + prog: prog, + buildConf: &Config{Goos: "linux", Goarch: "amd64"}, + crossCompile: crosscompile.Export{LLVMTarget: "x86_64-unknown-linux"}, + } + } + fingerprint := func(prog llssa.Program) (*aPackage, manifestData) { + pkg := newPackage() + if err := newContext(prog).collectFingerprint(pkg); err != nil { + t.Fatal(err) + } + data, err := decodeManifest(pkg.Manifest) + if err != nil { + t.Fatal(err) + } + return pkg, data + } + + plain, plainManifest := fingerprint(llssa.NewProgram(nil)) + nativeProg := llssa.NewProgram(nil) + nativeProg.SetLocalityInfo("example.com/state.value", llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + nativeProg.SetLocalStorage("example.com/state.value", llssa.LocalStorageNativeTLS) + native, nativeManifest := fingerprint(nativeProg) + contextProg := llssa.NewProgram(nil) + contextProg.SetLocalityInfo("example.com/state.value", llssa.LocalityInfo{Locality: llssa.GoroutineLocal}) + contextProg.SetLocalStorage("example.com/state.value", llssa.LocalStoragePackage) + withContext, contextManifest := fingerprint(contextProg) + initializedProg := llssa.NewProgram(nil) + initializedProg.SetLocalityInfo("example.com/state.value", llssa.LocalityInfo{ + Locality: llssa.ThreadLocal, + HasInitializer: true, + InitFunc: "example.com/state.__llgo_local_init_0", + InitOrder: 1, + }) + initializedProg.SetLocalStorage("example.com/state.value", llssa.LocalStorageNativeTLS) + initialized, initializedManifest := fingerprint(initializedProg) + + if plain.Fingerprint != native.Fingerprint { + t.Fatal("native TLS changed the package cache fingerprint") + } + if withContext.Fingerprint == plain.Fingerprint { + t.Fatal("local-context and plain builds shared a package cache fingerprint") + } + if initialized.Fingerprint == plain.Fingerprint { + t.Fatal("initialized native TLS and plain builds shared a package cache fingerprint") + } + if plainManifest.Common.LocalContext || nativeManifest.Common.LocalContext { + t.Fatal("plain or native-TLS manifest enabled the local context") + } + if !contextManifest.Common.LocalContext { + t.Fatal("context-backed locality was not recorded in the manifest") + } + if !initializedManifest.Common.LocalContext { + t.Fatal("native TLS initializer failure storage was not recorded in the manifest") + } +} + func TestDevLTOGlobalDCECollectFingerprint(t *testing.T) { td := t.TempDir() diff --git a/internal/build/fingerprint.go b/internal/build/fingerprint.go index 835b88df5e..ecc55298ab 100644 --- a/internal/build/fingerprint.go +++ b/internal/build/fingerprint.go @@ -121,6 +121,7 @@ type commonSection struct { EmitDWARF bool `yaml:"EMIT_DWARF,omitempty"` PCLNMode string `yaml:"PCLN_MODE,omitempty"` DisableBoundsChecks bool `yaml:"DISABLE_BOUNDS_CHECKS,omitempty"` + LocalContext bool `yaml:"LOCAL_CONTEXT,omitempty"` CC string `yaml:"CC,omitempty"` CCFlags []string `yaml:"CCFLAGS,omitempty"` CFlags []string `yaml:"CFLAGS,omitempty"` @@ -131,7 +132,9 @@ type commonSection struct { func (s *commonSection) empty() bool { return s.AbiMode == "" && len(s.BuildTags) == 0 && s.Target == "" && s.TargetABI == "" && - !s.GoGlobalDCE && !s.EnableLTOPlugin && !s.EmitDWARF && s.PCLNMode == "" && !s.DisableBoundsChecks && s.CC == "" && len(s.CCFlags) == 0 && len(s.CFlags) == 0 && len(s.LDFlags) == 0 && + !s.GoGlobalDCE && !s.EnableLTOPlugin && !s.EmitDWARF && s.PCLNMode == "" && + !s.DisableBoundsChecks && !s.LocalContext && + s.CC == "" && len(s.CCFlags) == 0 && len(s.CFlags) == 0 && len(s.LDFlags) == 0 && s.Linker == "" && len(s.ExtraFiles) == 0 } From 7508674a370e30f4d1e68bcd75a0654ac427d35e Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 14 Jul 2026 19:09:37 +0800 Subject: [PATCH 05/13] test: cover TLS and GLS runtime behavior --- cl/_testgo/localitycodegen/in.go | 74 +++ test/llgoext/locality_test.go | 544 ++++++++++++++++++ test/llgoext/testdata/localitybench/bench.go | 29 + .../testdata/localityfailure/failure.go | 57 ++ test/llgoext/testdata/localityscope/scope.go | 66 +++ 5 files changed, 770 insertions(+) create mode 100644 cl/_testgo/localitycodegen/in.go create mode 100644 test/llgoext/locality_test.go create mode 100644 test/llgoext/testdata/localitybench/bench.go create mode 100644 test/llgoext/testdata/localityfailure/failure.go create mode 100644 test/llgoext/testdata/localityscope/scope.go diff --git a/cl/_testgo/localitycodegen/in.go b/cl/_testgo/localitycodegen/in.go new file mode 100644 index 0000000000..8166bbfc50 --- /dev/null +++ b/cl/_testgo/localitycodegen/in.go @@ -0,0 +1,74 @@ +// LITTEST +package main + +// CHECK-DAG: @"{{.*}}localitycodegen.Scalar" = thread_local global i64 0 +// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_key" = global i8 0 +// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$guard" = thread_local global i8 0 +// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure" = global i8 0 +// CHECK-NOT: RegisterLocalRoot +// CHECK-NOT: localitycodegen.Pointer" = thread_local +// CHECK-NOT: localitycodegen.Initialized" = thread_local + +// CHECK-LABEL: define ptr @"{{.*}}localitycodegen.__llgo_local_block"() +// CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_key", i64 16, i64 8) +// CHECK: ret ptr + +// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init"() +// CHECK: call void @"{{.*}}localitycodegen.__llgo_local_init_0"() + +// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"() +// CHECK: load i8, ptr +// CHECK: call void @"{{.*}}runtime.EnsureLocalInitializer"(ptr @"{{.*}}localitycodegen.__llgo_tls_init$guard", ptr @"{{.*}}localitycodegen.__llgo_tls_init$failure" + +// CHECK-LABEL: define ptr @{{"?ExportedLocality"?}}() +// CHECK: call i64 @"{{.*}}EnterLocalContext" +// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"() +// CHECK: call void @"{{.*}}LeaveLocalContext" +// CHECK: ret ptr + +// CHECK-LABEL: define void @"{{.*}}localitycodegen.init"() +// CHECK: store i8 2, ptr +// CHECK: call ptr @"{{.*}}localitycodegen.newPointer"() +// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"() +// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"() + +// CHECK-LABEL: define { i64, ptr, ptr } @"{{.*}}localitycodegen.values"() +// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"() +// CHECK: load i64, ptr @"{{.*}}localitycodegen.Scalar" +// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"() +// CHECK: load ptr, ptr +// CHECK: load ptr, ptr + +// CHECK-LABEL: define ptr @"{{.*}}localitycodegen._llgo_routine$1"(ptr %0) +// CHECK: alloca %"{{.*}}LocalContext", align 8 +// CHECK: call i64 @"{{.*}}EnterLocalContext" +// CHECK: call void @"{{.*}}LeaveLocalContext" + +var backing int + +func newPointer() *int { + return &backing +} + +//llgo:tls +var Scalar int + +//llgo:gls +var Pointer *int + +//llgo:tls +var Initialized = newPointer() + +func values() (int, *int, *int) { + return Scalar, Pointer, Initialized +} + +//export ExportedLocality +func ExportedLocality() *int { + return Pointer +} + +func main() { + _, _, _ = values() + go values() +} diff --git a/test/llgoext/locality_test.go b/test/llgoext/locality_test.go new file mode 100644 index 0000000000..9dff66521d --- /dev/null +++ b/test/llgoext/locality_test.go @@ -0,0 +1,544 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package llgoext + +import ( + "runtime" + "sync/atomic" + "testing" + + "github.com/goplus/llgo/test/llgoext/testdata/localitybench" + "github.com/goplus/llgo/test/llgoext/testdata/localityfailure" + "github.com/goplus/llgo/test/llgoext/testdata/localityscope" +) + +var initializerSequence int + +func nextLocalValue(base int) int { + initializerSequence++ + return base + initializerSequence +} + +//llgo:tls +var tlsCounter int + +//llgo:gls +var glsCounter int + +//llgo:tls +var initializedTLS = nextLocalValue(100) + +//llgo:gls +var initializedGLS = nextLocalValue(200) + +type localitySnapshot struct { + tlsCounter int + glsCounter int + initializedTLS int + initializedGLS int +} + +func snapshotLocality() localitySnapshot { + return localitySnapshot{ + tlsCounter: tlsCounter, + glsCounter: glsCounter, + initializedTLS: initializedTLS, + initializedGLS: initializedGLS, + } +} + +func TestTLSAndGLSIsolation(t *testing.T) { + parentInitial := snapshotLocality() + if parentInitial.initializedTLS <= 100 || parentInitial.initializedGLS <= 200 { + t.Fatalf("parent initializers did not run: %+v", parentInitial) + } + + tlsCounter = 11 + glsCounter = 22 + parentSet := snapshotLocality() + + type childResult struct { + first localitySnapshot + again localitySnapshot + set localitySnapshot + } + done := make(chan childResult) + go func() { + first := snapshotLocality() + again := snapshotLocality() + tlsCounter = 31 + glsCounter = 32 + done <- childResult{first: first, again: again, set: snapshotLocality()} + }() + child := <-done + + if child.first.tlsCounter != 0 || child.first.glsCounter != 0 { + t.Fatalf("child inherited parent local values: %+v", child.first) + } + if child.first != child.again { + t.Fatalf("child initializer ran more than once: first=%+v again=%+v", child.first, child.again) + } + if child.first.initializedTLS == parentInitial.initializedTLS || child.first.initializedGLS == parentInitial.initializedGLS { + t.Fatalf("child reused parent initialization: parent=%+v child=%+v", parentInitial, child.first) + } + if child.set.tlsCounter != 31 || child.set.glsCounter != 32 { + t.Fatalf("child local writes were lost: %+v", child.set) + } + if got := snapshotLocality(); got != parentSet { + t.Fatalf("child changed parent local values: got=%+v want=%+v", got, parentSet) + } +} + +func TestPanickingInitializerIsSticky(t *testing.T) { + for attempt := 0; attempt < 2; attempt++ { + var recovered any + func() { + defer func() { recovered = recover() }() + _ = localityfailure.Value() + }() + if recovered == nil { + t.Fatalf("attempt %d did not re-panic", attempt) + } + if recovered != localityfailure.Failure { + t.Fatalf("attempt %d panic = %v, want %v", attempt, recovered, localityfailure.Failure) + } + } + if attempts := localityfailure.Attempts(); attempts != 2 { + t.Fatalf("initializer attempts = %d, want 2", attempts) + } +} + +func TestNilPanickingInitializerIsSticky(t *testing.T) { + for attempt := 0; attempt < 2; attempt++ { + var recovered any + func() { + defer func() { recovered = recover() }() + _ = localityfailure.NilValue() + }() + if recovered == nil { + t.Fatalf("attempt %d did not re-panic", attempt) + } + } + if attempts := localityfailure.NilAttempts(); attempts != 2 { + t.Fatalf("initializer attempts = %d, want 2", attempts) + } +} + +type recursiveInitializer interface { + value() int +} + +type recursiveSourceValue struct{} + +func (recursiveSourceValue) value() int { + return recursiveValue + 1 +} + +var recursiveSource recursiveInitializer = recursiveSourceValue{} + +//llgo:gls +var recursiveValue = recursiveSource.value() + +func TestRecursiveInitializerObservesPartialValue(t *testing.T) { + if recursiveValue != 1 { + t.Fatalf("recursive initializer value = %d, want 1", recursiveValue) + } + if recursiveValue != 1 { + t.Fatal("recursive initializer ran more than once") + } +} + +var lateInitializerAttempts int + +func nextLateValue() int { + lateInitializerAttempts++ + return 100 + lateInitializerAttempts +} + +// zLate sorts after the package init function in SSA member order. +// +//llgo:tls +var zLate = nextLateValue() + +func TestLateSortedInitializer(t *testing.T) { + value, attempts := zLate, lateInitializerAttempts + if value != 100+attempts { + t.Fatalf("late initializer value = %d, attempts = %d", value, attempts) + } + if again := zLate; again != value || lateInitializerAttempts != attempts { + t.Fatalf("late initializer repeated: value=%d/%d attempts=%d/%d", again, value, lateInitializerAttempts, attempts) + } +} + +type rootedValue struct { + value int + pad [256]byte +} + +func newRootedValue() *rootedValue { + return &rootedValue{value: 73} +} + +//llgo:gls +var rootedPointer = newRootedValue() + +//go:noinline +func touchRootedPointer() { + if rootedPointer == nil || rootedPointer.value != 73 { + panic("invalid rooted pointer") + } +} + +//go:noinline +func collectWithoutLocalPointer() { + for i := 0; i < 3; i++ { + _ = make([]byte, 1<<20) + runtime.GC() + } +} + +//go:noinline +func readRootedPointer() int { + return rootedPointer.value +} + +func TestLocalPointerIsGCRoot(t *testing.T) { + touchRootedPointer() + collectWithoutLocalPointer() + if got := readRootedPointer(); got != 73 { + t.Fatalf("rooted pointer value = %d, want 73", got) + } +} + +func TestLocalContextCleanupAfterThreadExit(t *testing.T) { + for i := 0; i < 16; i++ { + exited := make(chan struct{}) + go func() { + defer close(exited) + touchRootedPointer() + }() + <-exited + } + runtime.GC() + runtime.GC() +} + +//llgo:gls +var atomicGLS int64 + +func TestLocalAddressAndAtomicSemantics(t *testing.T) { + first := &atomicGLS + second := &atomicGLS + if first != second { + t.Fatalf("repeated GLS address changed: %p != %p", first, second) + } + atomic.StoreInt64(first, 7) + if got := atomic.AddInt64(&atomicGLS, 5); got != 12 { + t.Fatalf("atomic GLS value = %d, want 12", got) + } +} + +func localClosure() func() int { + glsCounter = 40 + return func() int { + glsCounter++ + return glsCounter + } +} + +func TestClosureUsesInvocationContext(t *testing.T) { + closure := localClosure() + done := make(chan int) + go func() { + done <- closure() + }() + if got := <-done; got != 1 { + t.Fatalf("closure used creator GLS value: got %d, want 1", got) + } + if glsCounter != 40 { + t.Fatalf("child closure changed parent GLS value: %d", glsCounter) + } +} + +type escapedBlockValue struct { + pointer *int + value int +} + +//llgo:gls +var escapedBlock escapedBlockValue + +func TestEscapedPackageBlockAddressSurvivesOwnerExit(t *testing.T) { + addresses := make(chan *escapedBlockValue) + exited := make(chan struct{}) + go func() { + value := 71 + escapedBlock = escapedBlockValue{pointer: &value, value: 71} + addresses <- &escapedBlock + close(exited) + }() + address := <-addresses + <-exited + runtime.GC() + runtime.GC() + if address.value != 71 || address.pointer == nil || *address.pointer != 71 { + t.Fatalf("escaped package block after owner exit = %+v", address) + } + done := make(chan bool) + go func(block *escapedBlockValue) { + block.value = 72 + *block.pointer = 72 + done <- true + }(address) + <-done + if address.value != 72 || *address.pointer != 72 { + t.Fatalf("escaped package block after cross-goroutine write = %+v", address) + } +} + +func TestEscapedPackageBlockAddressSurvivesGoexit(t *testing.T) { + addresses := make(chan *escapedBlockValue) + exited := make(chan struct{}) + go func() { + value := 81 + escapedBlock = escapedBlockValue{pointer: &value, value: 81} + address := &escapedBlock + defer close(exited) + defer func() { + addresses <- address + }() + runtime.Goexit() + }() + address := <-addresses + <-exited + runtime.GC() + if address.value != 81 || address.pointer == nil || *address.pointer != 81 { + t.Fatalf("escaped package block after Goexit = %+v", address) + } +} + +//llgo:gls +var zeroSizedGLS struct{} + +func TestZeroSizedNativeLocalAddressIsStable(t *testing.T) { + first := &zeroSizedGLS + second := &zeroSizedGLS + if first == nil || first != second { + t.Fatalf("zero-sized GLS address changed: %p != %p", first, second) + } +} + +func TestInitializerScopeRunsOncePerPackageKind(t *testing.T) { + firstBefore := localityscope.FirstCalls() + secondBefore := localityscope.SecondCalls() + type result struct { + firstValue int + firstCalls int + secondCalls int + firstCallsAgain int + secondValue int + secondCallsAfter int + } + done := make(chan result) + go func() { + firstValue := localityscope.First + firstCalls := localityscope.FirstCalls() + secondCalls := localityscope.SecondCalls() + _ = localityscope.First + firstCallsAgain := localityscope.FirstCalls() + secondValue := localityscope.Second + done <- result{ + firstValue: firstValue, + firstCalls: firstCalls, + secondCalls: secondCalls, + firstCallsAgain: firstCallsAgain, + secondValue: secondValue, + secondCallsAfter: localityscope.SecondCalls(), + } + }() + got := <-done + if got.firstValue == 0 || got.secondValue == 0 { + t.Fatalf("lazy initializer values = %+v", got) + } + if got.firstCalls != firstBefore+1 || got.firstCallsAgain != got.firstCalls { + t.Fatalf("first initializer calls = %+v, baseline %d", got, firstBefore) + } + if got.secondCalls != secondBefore+1 || got.secondCallsAfter != got.secondCalls { + t.Fatalf("package GLS initializers did not run together once: %+v, baseline %d", got, secondBefore) + } +} + +func TestMultiValueInitializerUsesOneGroup(t *testing.T) { + before := localityscope.PairCalls() + type result struct { + first, second int + afterFirst int + afterSecond int + } + done := make(chan result) + go func() { + first := localityscope.PairFirst + afterFirst := localityscope.PairCalls() + second := localityscope.PairSecond + done <- result{first, second, afterFirst, localityscope.PairCalls()} + }() + got := <-done + if got.first == 0 || got.second == 0 || got.afterFirst != before+1 || got.afterSecond != got.afterFirst { + t.Fatalf("multi-value initializer group = %+v, baseline %d", got, before) + } +} + +func TestCrossPackageMixedInitializerGroup(t *testing.T) { + before := localityscope.MixedCalls() + type result struct { + scalar int + pointer *int + addressStable bool + calls int + } + done := make(chan result) + go func() { + scalar := localityscope.MixedScalar + address := &localityscope.MixedScalar + done <- result{ + scalar: scalar, + pointer: localityscope.MixedPointer, + addressStable: address == &localityscope.MixedScalar, + calls: localityscope.MixedCalls(), + } + }() + got := <-done + if got.scalar == 0 || got.pointer == nil || !got.addressStable || got.calls != before+1 { + t.Fatalf("cross-package mixed initializer = %+v, baseline %d", got, before) + } +} + +var benchmarkOrdinary int + +//llgo:tls +var benchmarkTLS int + +//llgo:gls +var benchmarkGLS int + +var benchmarkSink int + +//go:noinline +func bumpOrdinaryGlobal() int { + benchmarkOrdinary++ + return benchmarkOrdinary +} + +//go:noinline +func bumpNativeTLS() int { + benchmarkTLS++ + return benchmarkTLS +} + +//go:noinline +func bumpNativeGLS() int { + benchmarkGLS++ + return benchmarkGLS +} + +type benchmarkPackageValue struct { + pointer *int + value int +} + +//llgo:tls +var benchmarkTLSPackage benchmarkPackageValue + +//llgo:gls +var benchmarkGLSPackage benchmarkPackageValue + +//go:noinline +func bumpTLSPackageBlock() int { + benchmarkTLSPackage.value++ + return benchmarkTLSPackage.value +} + +//go:noinline +func bumpGLSPackageBlock() int { + benchmarkGLSPackage.value++ + return benchmarkGLSPackage.value +} + +func BenchmarkOrdinaryGlobal(b *testing.B) { + value := 0 + for i := 0; i < b.N; i++ { + value += bumpOrdinaryGlobal() + } + benchmarkSink = value +} + +func BenchmarkNativeTLS(b *testing.B) { + value := 0 + for i := 0; i < b.N; i++ { + value += bumpNativeTLS() + } + benchmarkSink = value +} + +func BenchmarkNativeGLS(b *testing.B) { + value := 0 + for i := 0; i < b.N; i++ { + value += bumpNativeGLS() + } + benchmarkSink = value +} + +func BenchmarkTLSPackageBlock(b *testing.B) { + benchmarkTLSPackage.pointer = &benchmarkSink + b.ResetTimer() + value := 0 + for i := 0; i < b.N; i++ { + value += bumpTLSPackageBlock() + } + benchmarkSink = value +} + +func BenchmarkGLSPackageBlock(b *testing.B) { + benchmarkGLSPackage.pointer = &benchmarkSink + b.ResetTimer() + value := 0 + for i := 0; i < b.N; i++ { + value += bumpGLSPackageBlock() + } + benchmarkSink = value +} + +func BenchmarkGoroutineEntry(b *testing.B) { + for i := 0; i < b.N; i++ { + done := make(chan struct{}) + go func() { close(done) }() + <-done + } +} + +func BenchmarkGoroutinePackageBlockFirstTouch(b *testing.B) { + for i := 0; i < b.N; i++ { + done := make(chan struct{}) + go func() { + localitybench.Touch() + close(done) + }() + <-done + } +} diff --git a/test/llgoext/testdata/localitybench/bench.go b/test/llgoext/testdata/localitybench/bench.go new file mode 100644 index 0000000000..b9a1426252 --- /dev/null +++ b/test/llgoext/testdata/localitybench/bench.go @@ -0,0 +1,29 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localitybench + +var backing int + +//llgo:gls +var pointer *int + +//go:noinline +func Touch() { + pointer = &backing +} diff --git a/test/llgoext/testdata/localityfailure/failure.go b/test/llgoext/testdata/localityfailure/failure.go new file mode 100644 index 0000000000..c42990ebd8 --- /dev/null +++ b/test/llgoext/testdata/localityfailure/failure.go @@ -0,0 +1,57 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localityfailure + +const Failure = "locality initializer failed" + +var attempts int + +func initialize() int { + attempts++ + if attempts > 1 { + panic(Failure) + } + return attempts +} + +//llgo:tls +var value = initialize() + +//go:noinline +func Value() int { return value } + +func Attempts() int { return attempts } + +var nilAttempts int + +func initializeNil() int { + nilAttempts++ + if nilAttempts > 1 { + panic(nil) + } + return nilAttempts +} + +//llgo:gls +var nilValue = initializeNil() + +//go:noinline +func NilValue() int { return nilValue } + +func NilAttempts() int { return nilAttempts } diff --git a/test/llgoext/testdata/localityscope/scope.go b/test/llgoext/testdata/localityscope/scope.go new file mode 100644 index 0000000000..dce9828dca --- /dev/null +++ b/test/llgoext/testdata/localityscope/scope.go @@ -0,0 +1,66 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localityscope + +var firstCalls int +var secondCalls int + +func initFirst() int { + firstCalls++ + return 100 + firstCalls +} + +func initSecond() int { + secondCalls++ + return 200 + secondCalls +} + +//llgo:gls +var First = initFirst() + +//llgo:gls +var Second = initSecond() + +func FirstCalls() int { return firstCalls } +func SecondCalls() int { return secondCalls } + +var pairCalls int + +func initPair() (int, int) { + pairCalls++ + return 300 + pairCalls, 400 + pairCalls +} + +//llgo:gls +var PairFirst, PairSecond = initPair() + +func PairCalls() int { return pairCalls } + +var mixedCalls int +var mixedBacking = 500 + +func initMixed() (int, *int) { + mixedCalls++ + return 600 + mixedCalls, &mixedBacking +} + +//llgo:tls +var MixedScalar, MixedPointer = initMixed() + +func MixedCalls() int { return mixedCalls } From eed9fc29f605bd02e9f462487b2cfc0f2f54ea2a Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 00:53:54 +0800 Subject: [PATCH 06/13] runtime: split local package fast path --- runtime/internal/runtime/local_context.go | 14 +++++-- test/llgoext/runtime_g_bench_test.go | 45 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 test/llgoext/runtime_g_bench_test.go diff --git a/runtime/internal/runtime/local_context.go b/runtime/internal/runtime/local_context.go index 95f6226f8f..54ae9e408c 100644 --- a/runtime/internal/runtime/local_context.go +++ b/runtime/internal/runtime/local_context.go @@ -83,6 +83,17 @@ func releaseLocalBlocks(ctx *LocalContext) { // head fast path; other accesses move the matching block to the front. func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) + if ctx != nil { + first := ctx.blocks + if first != nil && first.key == key && align != 0 && align&(align-1) == 0 { + return localBlockData(first, align) + } + } + return localPackageSlow(ctx, key, size, align) +} + +//go:noinline +func localPackageSlow(ctx *LocalContext, key unsafe.Pointer, size, align uintptr) unsafe.Pointer { if ctx == nil { panic("runtime: local variable accessed outside a Go entry context") } @@ -93,9 +104,6 @@ func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { panic("runtime: invalid local package alignment") } first := ctx.blocks - if first != nil && first.key == key { - return localBlockData(first, align) - } var previous *localBlock for block := first; block != nil; block = block.next { if block.key == key { diff --git a/test/llgoext/runtime_g_bench_test.go b/test/llgoext/runtime_g_bench_test.go new file mode 100644 index 0000000000..927a75d3b9 --- /dev/null +++ b/test/llgoext/runtime_g_bench_test.go @@ -0,0 +1,45 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package llgoext + +import ( + "testing" + "unsafe" +) + +var runtimeDeferSink unsafe.Pointer + +//go:linkname runtimeGetThreadDefer github.com/goplus/llgo/runtime/internal/runtime.GetThreadDefer +func runtimeGetThreadDefer() unsafe.Pointer + +func BenchmarkRuntimeGetThreadDefer(b *testing.B) { + for i := 0; i < b.N; i++ { + runtimeDeferSink = runtimeGetThreadDefer() + } +} + +func BenchmarkRuntimeGoroutineEntryWithDefer(b *testing.B) { + for i := 0; i < b.N; i++ { + done := make(chan struct{}) + go func() { + defer close(done) + }() + <-done + } +} From d9f0a4000ce9a0e13f67f1c9344f9931825f0217 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 04:17:40 +0800 Subject: [PATCH 07/13] runtime: return local package data from hot block --- runtime/internal/runtime/local_context.go | 60 +++++++++++++---------- test/llgoext/locality_test.go | 18 +++++++ 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/runtime/internal/runtime/local_context.go b/runtime/internal/runtime/local_context.go index 54ae9e408c..9ce43dcdac 100644 --- a/runtime/internal/runtime/local_context.go +++ b/runtime/internal/runtime/local_context.go @@ -24,11 +24,15 @@ import "unsafe" // one-thread-per-goroutine backend maps both logical locality kinds to this one // physical package store. type LocalContext struct { - blocks *localBlock + // blocks points at the payload of the most recently used package block. + // Keeping the aligned payload at the head makes the common lookup return it + // directly; the block header is stored immediately before the payload. + blocks unsafe.Pointer } type localBlock struct { - next *localBlock + // next points at the next block's payload, not its header. + next unsafe.Pointer key unsafe.Pointer } @@ -67,14 +71,15 @@ func leaveCurrentLocalContext() { } func releaseLocalBlocks(ctx *LocalContext) { - block := ctx.blocks + data := ctx.blocks ctx.blocks = nil - for block != nil { + for data != nil { + block := localBlockHeader(data) next := block.next // Do not free block here: an address of a local variable may outlive its // owner. Breaking the links lets the GC retain only escaped blocks. block.next = nil - block = next + data = next } } @@ -84,9 +89,9 @@ func releaseLocalBlocks(ctx *LocalContext) { func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) if ctx != nil { - first := ctx.blocks - if first != nil && first.key == key && align != 0 && align&(align-1) == 0 { - return localBlockData(first, align) + firstData := ctx.blocks + if firstData != nil && localBlockHeader(firstData).key == key { + return firstData } } return localPackageSlow(ctx, key, size, align) @@ -103,24 +108,27 @@ func localPackageSlow(ctx *LocalContext, key unsafe.Pointer, size, align uintptr if align == 0 || align&(align-1) != 0 { panic("runtime: invalid local package alignment") } - first := ctx.blocks + firstData := ctx.blocks var previous *localBlock - for block := first; block != nil; block = block.next { + for data := firstData; data != nil; { + block := localBlockHeader(data) + next := block.next if block.key == key { - previous.next = block.next - block.next = first - ctx.blocks = block - return localBlockData(block, align) + previous.next = next + block.next = firstData + ctx.blocks = data + return data } previous = block + data = next } - block := newLocalBlock(key, size, align) - block.next = first - ctx.blocks = block - return localBlockData(block, align) + data := newLocalBlock(key, size, align) + localBlockHeader(data).next = firstData + ctx.blocks = data + return data } -func newLocalBlock(key unsafe.Pointer, size, align uintptr) *localBlock { +func newLocalBlock(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { header := unsafe.Sizeof(localBlock{}) padding := align - 1 if size == 0 { @@ -129,16 +137,16 @@ func newLocalBlock(key unsafe.Pointer, size, align uintptr) *localBlock { if header > ^uintptr(0)-padding || header+padding > ^uintptr(0)-size { panic("runtime: local package size overflow") } - block := (*localBlock)(AllocZ(header + padding + size)) - if block == nil { + allocation := AllocZ(header + padding + size) + if allocation == nil { panic("runtime: failed to allocate local package") } + data := unsafe.Pointer((uintptr(allocation) + header + padding) &^ padding) + block := localBlockHeader(data) block.key = key - return block + return data } -func localBlockData(block *localBlock, align uintptr) unsafe.Pointer { - padding := align - 1 - data := (uintptr(unsafe.Pointer(block)) + unsafe.Sizeof(localBlock{}) + padding) &^ padding - return unsafe.Pointer(data) +func localBlockHeader(data unsafe.Pointer) *localBlock { + return (*localBlock)(unsafe.Pointer(uintptr(data) - unsafe.Sizeof(localBlock{}))) } diff --git a/test/llgoext/locality_test.go b/test/llgoext/locality_test.go index 9dff66521d..3d49367531 100644 --- a/test/llgoext/locality_test.go +++ b/test/llgoext/locality_test.go @@ -105,6 +105,24 @@ func TestTLSAndGLSIsolation(t *testing.T) { } } +func TestLocalPackageMoveToFront(t *testing.T) { + type result struct { + local int + imported int + } + done := make(chan result) + go func() { + glsCounter = 41 + localityscope.First = 51 + glsCounter++ + localityscope.First++ + done <- result{local: glsCounter, imported: localityscope.First} + }() + if got := <-done; got != (result{local: 42, imported: 52}) { + t.Fatalf("local package values after move-to-front = %+v", got) + } +} + func TestPanickingInitializerIsSticky(t *testing.T) { for attempt := 0; attempt < 2; attempt++ { var recovered any From ef565feb39a8c172435514fcdbb0be5e99d585b3 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 04:51:58 +0800 Subject: [PATCH 08/13] test: cover locality debug metadata lowering --- cl/locality_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cl/locality_test.go b/cl/locality_test.go index 081e8b35e7..05fcad5b81 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -181,6 +181,36 @@ func values() (int, *int, int, *int) { } } +func TestLocalityDebugInfoOnlyUsesFixedGlobals(t *testing.T) { + EnableDebug(true) + EnableDbgSyms(true) + defer EnableDebug(false) + defer EnableDbgSyms(false) + _, ir := compileLocalitySource(t, `package locality + +//llgo:tls +var Direct int + +//llgo:gls +var Pointer *int + +func values() (int, *int) { return Direct, Pointer } +`) + + direct := `@"example.com/locality.Direct" = thread_local global i64` + start := strings.Index(ir, direct) + if start < 0 { + t.Fatalf("native TLS global not found:\n%s", ir) + } + end := strings.IndexByte(ir[start:], '\n') + if end < 0 || !strings.Contains(ir[start:start+end], "!dbg") { + t.Fatalf("native TLS global has no debug metadata:\n%s", ir[start:]) + } + if strings.Contains(ir, `@"example.com/locality.Pointer" =`) { + t.Fatalf("package-local pointer was emitted as a fixed debug global:\n%s", ir) + } +} + func TestLocalityInitializersPreserveGoOrderPerKind(t *testing.T) { _, ir := compileLocalitySource(t, `package locality From 91bc531ec07e775e171a69247e97cf52de3ff3c1 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 13:33:30 +0800 Subject: [PATCH 09/13] compiler: inline local package fast path --- cl/_testgo/localitycodegen/in.go | 9 ++ cl/locality_lower.go | 8 +- cl/locality_test.go | 54 +++++++++- ssa/local_context.go | 73 +++++++++++++- ssa/local_context_test.go | 100 +++++++++++++++++++ test/llgoext/locality_test.go | 41 ++++++++ test/llgoext/testdata/localitybench/bench.go | 28 ++++++ 7 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 ssa/local_context_test.go diff --git a/cl/_testgo/localitycodegen/in.go b/cl/_testgo/localitycodegen/in.go index 8166bbfc50..25927d2b47 100644 --- a/cl/_testgo/localitycodegen/in.go +++ b/cl/_testgo/localitycodegen/in.go @@ -5,11 +5,20 @@ package main // CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_key" = global i8 0 // CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$guard" = thread_local global i8 0 // CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure" = global i8 0 +// CHECK-DAG: @"{{.*}}runtime.currentLocalContext" = external thread_local global i64 // CHECK-NOT: RegisterLocalRoot // CHECK-NOT: localitycodegen.Pointer" = thread_local // CHECK-NOT: localitycodegen.Initialized" = thread_local // CHECK-LABEL: define ptr @"{{.*}}localitycodegen.__llgo_local_block"() +// CHECK: load i64, ptr @"{{.*}}runtime.currentLocalContext" +// CHECK: icmp ne i64 +// CHECK: load ptr, ptr +// CHECK: icmp ne ptr +// CHECK: sub i64 +// CHECK: load ptr, ptr +// CHECK: icmp eq ptr +// CHECK: ret ptr // CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_key", i64 16, i64 8) // CHECK: ret ptr diff --git a/cl/locality_lower.go b/cl/locality_lower.go index 10b6cd4cee..7dbea7a612 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -261,15 +261,11 @@ func (p *context) buildLocalPackage(pkg llssa.Package, owner *localPackage, defi owner.blockFunc = pkg.NewFunc(localitylayout.BlockName(owner.plan.Path), noArgResultSignature(result), llssa.InGo) owner.blockFunc.Inline(llssa.AlwaysInline) if define && !owner.blockFunc.HasBody() { - b := owner.blockFunc.MakeBody(1) - raw := b.Call( - pkg.RuntimeFunc("LocalPackage"), - b.Convert(p.prog.VoidPtr(), key.Expr), + owner.blockFunc.BuildLocalPackageAccessor( + key.Expr, p.prog.IntVal(p.prog.SizeOf(owner.typ), p.prog.Uintptr()), p.prog.IntVal(p.prog.AlignOf(owner.typ), p.prog.Uintptr()), ) - b.Return(b.Convert(p.prog.Pointer(owner.typ), raw)) - b.EndBuild() } } for _, kind := range []locality.Kind{locality.Thread, locality.Goroutine} { diff --git a/cl/locality_test.go b/cl/locality_test.go index 05fcad5b81..694058bd90 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -16,6 +16,10 @@ import ( ) func compileLocalitySource(t *testing.T, src string) (llssa.Program, string) { + return compileLocalitySourceWithNativeAnchor(t, src, true) +} + +func compileLocalitySourceWithNativeAnchor(t *testing.T, src string, nativeAnchor bool) (llssa.Program, string) { t.Helper() fset := token.NewFileSet() file, err := parser.ParseFile(fset, "locality.go", src, parser.ParseComments) @@ -32,6 +36,11 @@ func compileLocalitySource(t *testing.T, src string) (llssa.Program, string) { prog := ssatest.NewProgramEx(t, nil, imp) prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) prog.SetRuntime(localityRuntimePackage()) + if nativeAnchor { + anchor := llssa.PkgRuntime + ".currentLocalContext" + prog.SetLocalityInfo(anchor, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + prog.SetLocalStorage(anchor, llssa.LocalStorageNativeTLS) + } if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { t.Fatal(err) } @@ -62,9 +71,17 @@ func newLocalityTypeInfo() *types.Info { func localityRuntimePackage() *types.Package { pkg := types.NewPackage(llssa.PkgRuntime, "runtime") + unsafePointer := types.Typ[types.UnsafePointer] + blocks := types.NewField(token.NoPos, pkg, "blocks", unsafePointer, false) localContextName := types.NewTypeName(token.NoPos, pkg, "LocalContext", nil) - localContext := types.NewNamed(localContextName, types.NewStruct(nil, nil), nil) + localContext := types.NewNamed(localContextName, types.NewStruct([]*types.Var{blocks}, nil), nil) pkg.Scope().Insert(localContextName) + next := types.NewField(token.NoPos, pkg, "next", unsafePointer, false) + key := types.NewField(token.NoPos, pkg, "key", unsafePointer, false) + localBlockName := types.NewTypeName(token.NoPos, pkg, "localBlock", nil) + types.NewNamed(localBlockName, types.NewStruct([]*types.Var{next, key}, nil), nil) + pkg.Scope().Insert(localBlockName) + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, "currentLocalContext", types.Typ[types.Uintptr])) localPackageParams := types.NewTuple( types.NewParam(token.NoPos, pkg, "key", types.Typ[types.UnsafePointer]), @@ -94,6 +111,41 @@ func localityRuntimePackage() *types.Package { return pkg } +func TestLocalPackageAccessorUsesRuntimeAnchorStorage(t *testing.T) { + const src = `package locality + +//llgo:gls +var Pointer *int + +func value() *int { return Pointer } +` + for _, test := range []struct { + name string + nativeAnchor bool + declaration string + }{ + {"native TLS", true, "external thread_local global i64"}, + {"ordinary global", false, "external global i64"}, + } { + t.Run(test.name, func(t *testing.T) { + _, ir := compileLocalitySourceWithNativeAnchor(t, src, test.nativeAnchor) + anchor := `@"` + llssa.PkgRuntime + `.currentLocalContext" = ` + test.declaration + if !strings.Contains(ir, anchor) { + t.Fatalf("runtime context anchor declaration not found: %s\n%s", anchor, ir) + } + accessor := llvmFunction(t, ir, "example.com/locality.__llgo_local_block") + load := `load i64, ptr @"` + llssa.PkgRuntime + `.currentLocalContext"` + slow := `call ptr @"` + llssa.PkgRuntime + `.LocalPackage"` + if loadAt, slowAt := strings.Index(accessor, load), strings.Index(accessor, slow); loadAt < 0 || slowAt < 0 || loadAt >= slowAt { + t.Fatalf("accessor does not load the runtime anchor before its slow path:\n%s", accessor) + } + if got := strings.Count(accessor, "icmp "); got != 3 { + t.Fatalf("accessor comparisons = %d, want context/head/key checks:\n%s", got, accessor) + } + }) + } +} + func llvmFunction(t *testing.T, ir, name string) string { t.Helper() markerAt := strings.Index(ir, `@"`+name+`"(`) diff --git a/ssa/local_context.go b/ssa/local_context.go index 9739ff19cc..cec3e2878a 100644 --- a/ssa/local_context.go +++ b/ssa/local_context.go @@ -16,7 +16,16 @@ package ssa -import "go/types" +import ( + "go/token" + "go/types" +) + +const ( + runtimeCurrentLocalContext = "currentLocalContext" + runtimeLocalContext = "LocalContext" + runtimeLocalBlock = "localBlock" +) // EnterLocalContext creates the stack root used by TLS/GLS locality blocks and // installs it for the current outermost Go entry. previous is nonzero only for @@ -36,3 +45,65 @@ func (b Builder) EnterLocalContext() (ctx, previous Expr) { func (b Builder) LeaveLocalContext(ctx, previous Expr) { b.Call(b.Pkg.rtFunc("LeaveLocalContext"), ctx, previous) } + +// BuildLocalPackageAccessor builds an always-hot package block lookup around +// LocalPackage. Runtime Go types and locality metadata are the ABI: they supply +// the context/header layouts and whether the context anchor is native TLS or an +// ordinary global. The generated code therefore does not duplicate target +// selection or byte offsets. +func (p Function) BuildLocalPackageAccessor(key, size, align Expr) { + prog := p.Prog + runtimePkg := prog.runtime() + contextType := runtimePkg.Scope().Lookup(runtimeLocalContext).Type() + blockType := runtimePkg.Scope().Lookup(runtimeLocalBlock).Type() + _, contextBlocks, _ := types.LookupFieldOrMethod(contextType, true, runtimePkg, "blocks") + _, blockKey, _ := types.LookupFieldOrMethod(blockType, true, runtimePkg, "key") + + b := p.MakeBody(5) + checkHead := p.Block(1) + checkKey := p.Block(2) + hit := p.Block(3) + slow := p.Block(4) + key = b.Convert(prog.VoidPtr(), key) + + current := b.Load(p.Pkg.runtimeGlobal(runtimeCurrentLocalContext).Expr) + hasContext := b.BinOp(token.NEQ, current, prog.IntVal(0, prog.Uintptr())) + b.If(hasContext, checkHead, slow) + + b.SetBlock(checkHead) + context := b.Convert(prog.Pointer(prog.Type(contextType, InGo)), current) + head := b.Load(b.FieldAddr(context, contextBlocks[0])) + hasHead := b.BinOp(token.NEQ, head, prog.Nil(head.Type)) + b.If(hasHead, checkKey, slow) + + b.SetBlock(checkKey) + headerAddress := b.BinOp( + token.SUB, + b.Convert(prog.Uintptr(), head), + prog.IntVal(prog.SizeOf(prog.Type(blockType, InGo)), prog.Uintptr()), + ) + header := b.Convert(prog.Pointer(prog.Type(blockType, InGo)), headerAddress) + foundKey := b.Load(b.FieldAddr(header, blockKey[0])) + b.If(b.BinOp(token.EQL, foundKey, key), hit, slow) + + b.SetBlock(hit) + result := p.raw.Type.(*types.Signature).Results().At(0).Type() + b.Return(b.Convert(prog.rawType(result), head)) + + b.SetBlock(slow) + raw := b.Call(p.Pkg.rtFunc("LocalPackage"), key, size, align) + b.Return(b.Convert(prog.rawType(result), raw)) + b.EndBuild() +} + +func (p Package) runtimeGlobal(name string) Global { + p.NeedRuntime = true + runtimePkg := p.Prog.runtime() + variable := runtimePkg.Scope().Lookup(name).(*types.Var) + fullName := FullName(runtimePkg, name) + typ := types.NewPointer(variable.Type()) + if locality, ok := p.Prog.VariableLocality(fullName); ok && locality.LocalStorage == LocalStorageNativeTLS { + return p.NewThreadLocalVar(fullName, typ, InGo) + } + return p.NewVar(fullName, typ, InGo) +} diff --git a/ssa/local_context_test.go b/ssa/local_context_test.go new file mode 100644 index 0000000000..9517b13a9f --- /dev/null +++ b/ssa/local_context_test.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import ( + "go/token" + "go/types" + "strings" + "testing" +) + +func TestBuildLocalPackageAccessor(t *testing.T) { + for _, test := range []struct { + name string + nativeAnchor bool + declaration string + }{ + {"native TLS anchor", true, "external thread_local global i64"}, + {"ordinary anchor", false, "external global i64"}, + } { + t.Run(test.name, func(t *testing.T) { + prog := NewProgram(nil) + prog.SetRuntime(localContextTestRuntime()) + anchorName := PkgRuntime + "." + runtimeCurrentLocalContext + if test.nativeAnchor { + prog.SetLocalityInfo(anchorName, LocalityInfo{Locality: ThreadLocal}) + prog.SetLocalStorage(anchorName, LocalStorageNativeTLS) + } + + pkg := prog.NewPackage("accessor", "example.com/accessor") + key := pkg.NewVar("example.com/accessor.key", types.NewPointer(types.Typ[types.Uint8]), InGo) + key.InitNil() + field := types.NewField(token.NoPos, nil, "pointer", types.NewPointer(types.Typ[types.Int]), false) + block := types.NewStruct([]*types.Var{field}, nil) + blockType := prog.Type(block, InGo) + result := types.NewPointer(block) + results := types.NewTuple(types.NewVar(token.NoPos, nil, "", result)) + accessor := pkg.NewFunc("example.com/accessor.block", types.NewSignatureType(nil, nil, nil, nil, results, false), InGo) + accessor.BuildLocalPackageAccessor( + key.Expr, + prog.IntVal(prog.SizeOf(blockType), prog.Uintptr()), + prog.IntVal(prog.AlignOf(blockType), prog.Uintptr()), + ) + + ir := pkg.String() + anchor := `@"` + anchorName + `" = ` + test.declaration + if !strings.Contains(ir, anchor) { + t.Fatalf("context anchor declaration not found: %s\n%s", anchor, ir) + } + if got := strings.Count(ir, "icmp "); got != 3 { + t.Fatalf("accessor comparisons = %d, want context/head/key checks:\n%s", got, ir) + } + if !strings.Contains(ir, `call ptr @"`+PkgRuntime+`.LocalPackage"`) { + t.Fatalf("accessor has no LocalPackage slow path:\n%s", ir) + } + }) + } +} + +func localContextTestRuntime() *types.Package { + pkg := types.NewPackage(PkgRuntime, "runtime") + unsafePointer := types.Typ[types.UnsafePointer] + + contextName := types.NewTypeName(token.NoPos, pkg, runtimeLocalContext, nil) + contextFields := []*types.Var{types.NewField(token.NoPos, pkg, "blocks", unsafePointer, false)} + types.NewNamed(contextName, types.NewStruct(contextFields, nil), nil) + pkg.Scope().Insert(contextName) + + blockName := types.NewTypeName(token.NoPos, pkg, runtimeLocalBlock, nil) + blockFields := []*types.Var{ + types.NewField(token.NoPos, pkg, "next", unsafePointer, false), + types.NewField(token.NoPos, pkg, "key", unsafePointer, false), + } + types.NewNamed(blockName, types.NewStruct(blockFields, nil), nil) + pkg.Scope().Insert(blockName) + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, runtimeCurrentLocalContext, types.Typ[types.Uintptr])) + + params := types.NewTuple( + types.NewVar(token.NoPos, pkg, "key", unsafePointer), + types.NewVar(token.NoPos, pkg, "size", types.Typ[types.Uintptr]), + types.NewVar(token.NoPos, pkg, "align", types.Typ[types.Uintptr]), + ) + results := types.NewTuple(types.NewVar(token.NoPos, pkg, "", unsafePointer)) + pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "LocalPackage", types.NewSignatureType(nil, nil, nil, params, results, false))) + return pkg +} diff --git a/test/llgoext/locality_test.go b/test/llgoext/locality_test.go index 3d49367531..796f320d47 100644 --- a/test/llgoext/locality_test.go +++ b/test/llgoext/locality_test.go @@ -456,6 +456,7 @@ var benchmarkTLS int var benchmarkGLS int var benchmarkSink int +var benchmarkReadSink uintptr //go:noinline func bumpOrdinaryGlobal() int { @@ -542,6 +543,46 @@ func BenchmarkGLSPackageBlock(b *testing.B) { benchmarkSink = value } +func TestComparableLocalityReads(t *testing.T) { + localitybench.PrepareReads() + ordinary := localitybench.ReadOrdinaryGlobal() + native := localitybench.ReadNativeTLS() + local := localitybench.ReadGLSPackage() + if ordinary == 0 || native != ordinary || local != ordinary { + t.Fatalf("comparable locality reads = ordinary:%#x native:%#x GLS:%#x", ordinary, native, local) + } +} + +func BenchmarkComparableOrdinaryGlobalRead(b *testing.B) { + localitybench.PrepareReads() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localitybench.ReadOrdinaryGlobal() + } + benchmarkReadSink = value +} + +func BenchmarkComparableNativeTLSRead(b *testing.B) { + localitybench.PrepareReads() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localitybench.ReadNativeTLS() + } + benchmarkReadSink = value +} + +func BenchmarkComparableGLSPackageRead(b *testing.B) { + localitybench.PrepareReads() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localitybench.ReadGLSPackage() + } + benchmarkReadSink = value +} + func BenchmarkGoroutineEntry(b *testing.B) { for i := 0; i < b.N; i++ { done := make(chan struct{}) diff --git a/test/llgoext/testdata/localitybench/bench.go b/test/llgoext/testdata/localitybench/bench.go index b9a1426252..2514a55b33 100644 --- a/test/llgoext/testdata/localitybench/bench.go +++ b/test/llgoext/testdata/localitybench/bench.go @@ -18,8 +18,15 @@ package localitybench +import "unsafe" + var backing int +var ordinaryPointer *int + +//llgo:tls +var nativePointerBits uintptr + //llgo:gls var pointer *int @@ -27,3 +34,24 @@ var pointer *int func Touch() { pointer = &backing } + +func PrepareReads() { + ordinaryPointer = &backing + nativePointerBits = uintptr(unsafe.Pointer(&backing)) + pointer = &backing +} + +//go:noinline +func ReadOrdinaryGlobal() uintptr { + return uintptr(unsafe.Pointer(ordinaryPointer)) +} + +//go:noinline +func ReadNativeTLS() uintptr { + return nativePointerBits +} + +//go:noinline +func ReadGLSPackage() uintptr { + return uintptr(unsafe.Pointer(pointer)) +} From 88be3e71142f00caf6649ce9064d88700ccc56d7 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 14:51:15 +0800 Subject: [PATCH 10/13] runtime: publish FuncForPC cache entries atomically Store each immutable Func as one atomic pointer and validate the queried PC from the Func itself. This prevents concurrent cache replacement from combining a PC and function from different writers. Apply the same publication rule to the prebuilt function cache and stress multiple PCs during concurrent first use. --- .../lib/runtime/pprof_runtime_stub_llgo.go | 49 ++++++++++++------- runtime/internal/lib/runtime/symtab.go | 12 ++--- test/go/runtime_lineinfo_stack_test.go | 48 ++++++++++++++++-- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 8885a25458..57dba0f3db 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -5,6 +5,7 @@ package runtime import ( "unsafe" + latomic "github.com/goplus/llgo/runtime/internal/lib/sync/atomic" llrt "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -100,26 +101,36 @@ func SetCPUProfileRate(hz int) {} const funcForPCCacheSets = 1024 const funcForPCCacheWays = 4 -type funcForPCCacheEntry struct { - pc uintptr - fn *Func -} +// Cache immutable Func pointers as single atomic words. The queried pc lives +// in Func itself, so concurrent replacement cannot combine fields from two +// different cache entries. +var funcForPCCache [funcForPCCacheSets][funcForPCCacheWays]unsafe.Pointer +var funcForPCCacheNext [funcForPCCacheSets]uint32 +var funcForPCLast unsafe.Pointer -var funcForPCCache [funcForPCCacheSets][funcForPCCacheWays]funcForPCCacheEntry -var funcForPCCacheNext [funcForPCCacheSets]uint8 -var funcForPCLast funcForPCCacheEntry +func cachedFuncForPC(entry *unsafe.Pointer, pc uintptr) *Func { + p := latomic.LoadPointer(entry) + if p == nil { + return nil + } + fn := (*Func)(p) + if fn.pc != pc { + return nil + } + return fn +} func FuncForPC(pc uintptr) *Func { // External metadata must be installed before consulting either cache: // caching a pre-load miss would otherwise survive a successful load. ensureRuntimePCLN() - if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { + if fn := cachedFuncForPC(&funcForPCLast, pc); fn != nil { return fn } set := &funcForPCCache[funcForPCCacheIndex(pc)] for i := 0; i < funcForPCCacheWays; i++ { - if fn := set[i].fn; fn != nil && set[i].pc == pc { - funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} + if fn := cachedFuncForPC(&set[i], pc); fn != nil { + latomic.StorePointer(&funcForPCLast, unsafe.Pointer(fn)) return fn } } @@ -235,12 +246,12 @@ func newFuncForPC(pc uintptr, sym pcSymbol) *Func { // symbolized, going through the FuncForPC cache so repeated CallersFrames // walks over the same PCs stop allocating a Func per frame. func frameFuncForPC(pc uintptr, sym pcSymbol, name string) *Func { - if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { + if fn := cachedFuncForPC(&funcForPCLast, pc); fn != nil { return fn } set := &funcForPCCache[funcForPCCacheIndex(pc)] for i := 0; i < funcForPCCacheWays; i++ { - if fn := set[i].fn; fn != nil && set[i].pc == pc { + if fn := cachedFuncForPC(&set[i], pc); fn != nil { return fn } } @@ -264,16 +275,16 @@ func cacheFuncForPC(pc uintptr, fn *Func) { setIndex := funcForPCCacheIndex(pc) set := &funcForPCCache[setIndex] for i := 0; i < funcForPCCacheWays; i++ { - if set[i].fn == nil || set[i].pc == pc { - set[i] = funcForPCCacheEntry{pc: pc, fn: fn} - funcForPCLast = set[i] + p := latomic.LoadPointer(&set[i]) + if p == nil || (*Func)(p).pc == pc { + latomic.StorePointer(&set[i], unsafe.Pointer(fn)) + latomic.StorePointer(&funcForPCLast, unsafe.Pointer(fn)) return } } - way := funcForPCCacheNext[setIndex] & (funcForPCCacheWays - 1) - funcForPCCacheNext[setIndex] = way + 1 - set[way] = funcForPCCacheEntry{pc: pc, fn: fn} - funcForPCLast = set[way] + way := (latomic.AddUint32(&funcForPCCacheNext[setIndex], 1) - 1) & (funcForPCCacheWays - 1) + latomic.StorePointer(&set[way], unsafe.Pointer(fn)) + latomic.StorePointer(&funcForPCLast, unsafe.Pointer(fn)) } func funcForPCCacheIndex(pc uintptr) uintptr { diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index d4b7f100f6..f9c834c4ad 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -805,22 +805,22 @@ var runtimePrebuiltEntriesOnce uint32 // lookups. The set-associative pc cache in FuncForPC thrashes once the live // pc population outgrows it (thousands of distinct functions queried in a // loop); this cache is keyed by table row, so batch workloads stay O(search) -// after the first pass regardless of scale. Same benign-race model as the -// pc cache: word-sized pointer stores of identical values. +// after the first pass regardless of scale. Atomic pointer publication keeps +// concurrently created Func values fully initialized before readers use them. var runtimePrebuiltFuncs []unsafe.Pointer func prebuiltFuncCacheLoad(idx int) unsafe.Pointer { if idx < 0 || idx >= len(runtimePrebuiltFuncs) { return nil } - return runtimePrebuiltFuncs[idx] + return latomic.LoadPointer(&runtimePrebuiltFuncs[idx]) } func prebuiltFuncCacheStore(idx int, fn unsafe.Pointer) { if idx < 0 || idx >= len(runtimePrebuiltFuncs) { return } - runtimePrebuiltFuncs[idx] = fn + latomic.StorePointer(&runtimePrebuiltFuncs[idx], fn) } // prebuiltFrameIndexForEntry returns the ftab row whose entry is exactly pc, @@ -1573,9 +1573,9 @@ func init() { // Write-warm the FuncForPC cache: its first stores otherwise take // zero-fill write faults, one per page, on the first few lookups. for i := 0; i < funcForPCCacheSets; i += 4096 / int(unsafe.Sizeof(funcForPCCache[0])) { - funcForPCCache[i][0].pc = 0 + latomic.StorePointer(&funcForPCCache[i][0], nil) } - funcForPCCache[funcForPCCacheSets-1][0].pc = 0 + latomic.StorePointer(&funcForPCCache[funcForPCCacheSets-1][0], nil) } func coldFuncInfoEntryLookup(pc uintptr) (pcSymbol, bool) { diff --git a/test/go/runtime_lineinfo_stack_test.go b/test/go/runtime_lineinfo_stack_test.go index 52fffcabef..673276bcd7 100644 --- a/test/go/runtime_lineinfo_stack_test.go +++ b/test/go/runtime_lineinfo_stack_test.go @@ -224,6 +224,7 @@ func TestRuntimeLineInfoAndStack(t *testing.T) { const runtimeFuncInfoConcurrentFirstUseProbe = `package main import ( + "reflect" "runtime" "strconv" "strings" @@ -232,16 +233,33 @@ import ( func main() { const n = 32 + const rounds = 1000 start := make(chan struct{}) errc := make(chan string, n) var wg sync.WaitGroup for i := 0; i < n; i++ { + target := concurrentTargets[i%len(concurrentTargets)] + pc := reflect.ValueOf(target.fn).Pointer() wg.Add(1) - go func() { + go func(target concurrentTarget, pc uintptr) { defer wg.Done() <-start - errc <- checkRuntimeInfo() - }() + for j := 0; j < rounds; j++ { + fn := runtime.FuncForPC(pc) + if fn == nil || fn.Name() != target.name { + name := "" + if fn != nil { + name = fn.Name() + } + errc <- "bad target func: " + name + return + } + if err := checkRuntimeInfo(); err != "" { + errc <- err + return + } + } + }(target, pc) } close(start) wg.Wait() @@ -253,6 +271,30 @@ func main() { } } +type concurrentTarget struct { + fn func() + name string +} + +var concurrentTargets = []concurrentTarget{ + {concurrentTarget0, "main.concurrentTarget0"}, + {concurrentTarget1, "main.concurrentTarget1"}, + {concurrentTarget2, "main.concurrentTarget2"}, + {concurrentTarget3, "main.concurrentTarget3"}, +} + +//go:noinline +func concurrentTarget0() {} + +//go:noinline +func concurrentTarget1() {} + +//go:noinline +func concurrentTarget2() {} + +//go:noinline +func concurrentTarget3() {} + //go:noinline func checkRuntimeInfo() string { pc, file, line, ok := runtime.Caller(0) // CONCURRENT_CALLER_MARK From 3820a4ded1d60221571464cb0587e8b26c2d9ce0 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 15 Jul 2026 21:21:48 +0800 Subject: [PATCH 11/13] compiler/runtime: use direct package locality caches Give each package-backed locality block an owner-local TLS cache so hot accesses are O(1) regardless of the number of touched packages. Keep the block list only for GC reachability and teardown. Require locality variables to be unexported and reject go:linkname aliases. Add codegen, lifecycle, and 2/4/8-package performance coverage. --- cl/_testgo/localitycodegen/in.go | 34 +-- cl/locality_lower.go | 26 +- cl/locality_lower_test.go | 4 +- cl/locality_test.go | 252 +++++++----------- internal/build/build_test.go | 12 +- internal/locality/layout/layout.go | 10 +- internal/locality/layout/layout_test.go | 4 +- internal/locality/locality_test.go | 22 +- internal/locality/scan.go | 6 + runtime/internal/runtime/local_context.go | 62 ++--- .../internal/runtime/local_context_stub.go | 11 +- runtime/internal/runtime/local_initializer.go | 10 +- ssa/local_context.go | 69 +---- ssa/local_context_test.go | 84 ++---- ssa/locality.go | 69 +++-- ssa/locality_test.go | 78 ++---- test/llgoext/locality_test.go | 46 +++- test/llgoext/localitymulti/locality_test.go | 115 ++++++++ .../testdata/localityblocks/p0/block.go | 31 +++ .../testdata/localityblocks/p1/block.go | 31 +++ .../testdata/localityblocks/p2/block.go | 31 +++ .../testdata/localityblocks/p3/block.go | 31 +++ .../testdata/localityblocks/p4/block.go | 31 +++ .../testdata/localityblocks/p5/block.go | 31 +++ .../testdata/localityblocks/p6/block.go | 31 +++ .../testdata/localityblocks/p7/block.go | 31 +++ test/llgoext/testdata/localityscope/scope.go | 25 +- 27 files changed, 729 insertions(+), 458 deletions(-) create mode 100644 test/llgoext/localitymulti/locality_test.go create mode 100644 test/llgoext/testdata/localityblocks/p0/block.go create mode 100644 test/llgoext/testdata/localityblocks/p1/block.go create mode 100644 test/llgoext/testdata/localityblocks/p2/block.go create mode 100644 test/llgoext/testdata/localityblocks/p3/block.go create mode 100644 test/llgoext/testdata/localityblocks/p4/block.go create mode 100644 test/llgoext/testdata/localityblocks/p5/block.go create mode 100644 test/llgoext/testdata/localityblocks/p6/block.go create mode 100644 test/llgoext/testdata/localityblocks/p7/block.go diff --git a/cl/_testgo/localitycodegen/in.go b/cl/_testgo/localitycodegen/in.go index 25927d2b47..b200a72584 100644 --- a/cl/_testgo/localitycodegen/in.go +++ b/cl/_testgo/localitycodegen/in.go @@ -1,25 +1,19 @@ // LITTEST package main -// CHECK-DAG: @"{{.*}}localitycodegen.Scalar" = thread_local global i64 0 -// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_key" = global i8 0 +// CHECK-DAG: @"{{.*}}localitycodegen.scalar" = thread_local global i64 0 +// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_cache" = thread_local global i64 0 // CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$guard" = thread_local global i8 0 -// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure" = global i8 0 -// CHECK-DAG: @"{{.*}}runtime.currentLocalContext" = external thread_local global i64 +// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure_cache" = thread_local global i64 0 // CHECK-NOT: RegisterLocalRoot -// CHECK-NOT: localitycodegen.Pointer" = thread_local -// CHECK-NOT: localitycodegen.Initialized" = thread_local +// CHECK-NOT: localitycodegen.pointer" = thread_local +// CHECK-NOT: localitycodegen.initialized" = thread_local // CHECK-LABEL: define ptr @"{{.*}}localitycodegen.__llgo_local_block"() -// CHECK: load i64, ptr @"{{.*}}runtime.currentLocalContext" +// CHECK: load i64, ptr @"{{.*}}localitycodegen.__llgo_local_cache" // CHECK: icmp ne i64 -// CHECK: load ptr, ptr -// CHECK: icmp ne ptr -// CHECK: sub i64 -// CHECK: load ptr, ptr -// CHECK: icmp eq ptr // CHECK: ret ptr -// CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_key", i64 16, i64 8) +// CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_cache", i64 16, i64 8) // CHECK: ret ptr // CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init"() @@ -27,7 +21,7 @@ package main // CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"() // CHECK: load i8, ptr -// CHECK: call void @"{{.*}}runtime.EnsureLocalInitializer"(ptr @"{{.*}}localitycodegen.__llgo_tls_init$guard", ptr @"{{.*}}localitycodegen.__llgo_tls_init$failure" +// CHECK: call void @"{{.*}}runtime.EnsureLocalInitializer"(ptr @"{{.*}}localitycodegen.__llgo_tls_init$guard", ptr @"{{.*}}localitycodegen.__llgo_tls_init$failure_cache" // CHECK-LABEL: define ptr @{{"?ExportedLocality"?}}() // CHECK: call i64 @"{{.*}}EnterLocalContext" @@ -43,7 +37,7 @@ package main // CHECK-LABEL: define { i64, ptr, ptr } @"{{.*}}localitycodegen.values"() // CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"() -// CHECK: load i64, ptr @"{{.*}}localitycodegen.Scalar" +// CHECK: load i64, ptr @"{{.*}}localitycodegen.scalar" // CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"() // CHECK: load ptr, ptr // CHECK: load ptr, ptr @@ -60,21 +54,21 @@ func newPointer() *int { } //llgo:tls -var Scalar int +var scalar int //llgo:gls -var Pointer *int +var pointer *int //llgo:tls -var Initialized = newPointer() +var initialized = newPointer() func values() (int, *int, *int) { - return Scalar, Pointer, Initialized + return scalar, pointer, initialized } //export ExportedLocality func ExportedLocality() *int { - return Pointer + return pointer } func main() { diff --git a/cl/locality_lower.go b/cl/locality_lower.go index 7dbea7a612..b06dc2e432 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -44,10 +44,10 @@ type localPackage struct { } type localInitializer struct { - guard llssa.Global - failureKey llssa.Global - dispatch llssa.Function - ensure llssa.Function + guard llssa.Global + failureCache llssa.Global + dispatch llssa.Function + ensure llssa.Function } type localBaseCacheKey struct { @@ -131,9 +131,9 @@ func (p *context) prepareLocalVariables(pkg llssa.Package, globals []*ssa.Global } } -// localVariableFor resolves one Go SSA global to the canonical package plan. -// Both local definitions and imported references use this path so linkname and -// layout validation cannot diverge between the two lowering cases. +// localVariableFor validates one Go SSA global and finds its declaring package +// plan. Both definitions and references use this path so forbidden linkname +// aliases and layout validation cannot diverge between lowering cases. func (p *context) localVariableFor(pkg llssa.Package, global *ssa.Global, defineCurrent bool) (*localVariable, bool, error) { fullName := llssa.FullName(global.Pkg.Pkg, global.Name()) canonical, info, ok, err := p.prog.ResolveLocality(fullName) @@ -253,16 +253,16 @@ func (p *context) buildLocalPackage(pkg llssa.Package, owner *localPackage, defi } structType := types.NewStruct(fields, nil) owner.typ = p.prog.Type(structType, llssa.InGo) - key := pkg.NewVar(localitylayout.BlockKeyName(owner.plan.Path), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) + cache := pkg.NewThreadLocalVar(localitylayout.BlockCacheName(owner.plan.Path), types.NewPointer(types.Typ[types.Uintptr]), llssa.InGo) if define { - key.InitNil() + cache.InitNil() } result := types.NewPointer(structType) owner.blockFunc = pkg.NewFunc(localitylayout.BlockName(owner.plan.Path), noArgResultSignature(result), llssa.InGo) owner.blockFunc.Inline(llssa.AlwaysInline) if define && !owner.blockFunc.HasBody() { owner.blockFunc.BuildLocalPackageAccessor( - key.Expr, + cache.Expr, p.prog.IntVal(p.prog.SizeOf(owner.typ), p.prog.Uintptr()), p.prog.IntVal(p.prog.AlignOf(owner.typ), p.prog.Uintptr()), ) @@ -280,7 +280,7 @@ func (p *context) buildLocalPackage(pkg llssa.Package, owner *localPackage, defi func (p *context) buildLocalInitializer(pkg llssa.Package, owner *localPackage, kind locality.Kind, initializers []localitylayout.Initializer, define bool) *localInitializer { ret := &localInitializer{} ret.guard = pkg.NewThreadLocalVar(localitylayout.GuardName(owner.plan.Path, kind), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) - ret.failureKey = pkg.NewVar(localitylayout.FailureKeyName(owner.plan.Path, kind), types.NewPointer(types.Typ[types.Uint8]), llssa.InGo) + ret.failureCache = pkg.NewThreadLocalVar(localitylayout.FailureCacheName(owner.plan.Path, kind), types.NewPointer(types.Typ[types.Uintptr]), llssa.InGo) ret.dispatch = pkg.NewFunc(localitylayout.InitName(owner.plan.Path, kind), llssa.NoArgsNoRet, llssa.InGo) ret.ensure = pkg.NewFunc(localitylayout.EnsureName(owner.plan.Path, kind), llssa.NoArgsNoRet, llssa.InGo) ret.ensure.Inline(llssa.AlwaysInline) @@ -288,7 +288,7 @@ func (p *context) buildLocalInitializer(pkg llssa.Package, owner *localPackage, return ret } ret.guard.InitNil() - ret.failureKey.InitNil() + ret.failureCache.InitNil() if !ret.dispatch.HasBody() { b := ret.dispatch.MakeBody(1) for _, initializer := range initializers { @@ -307,7 +307,7 @@ func (p *context) buildLocalInitializer(pkg llssa.Package, owner *localPackage, b.Call( pkg.RuntimeFunc("EnsureLocalInitializer"), ret.guard.Expr, - b.Convert(p.prog.VoidPtr(), ret.failureKey.Expr), + ret.failureCache.Expr, closure, ) b.Jump(ret.ensure.Block(2)) diff --git a/cl/locality_lower_test.go b/cl/locality_lower_test.go index 7610d70c13..8d16ae9e26 100644 --- a/cl/locality_lower_test.go +++ b/cl/locality_lower_test.go @@ -124,13 +124,13 @@ func TestLocalityLoweringDiagnostics(t *testing.T) { } }) - t.Run("linkname cycle", func(t *testing.T) { + t.Run("local linkname", func(t *testing.T) { prog := newProgram() other := typesPkg.Path() + ".Other" prog.SetLinkname(name, other) prog.SetLinkname(other, name) ctx := &context{prog: prog, goTyps: typesPkg} - if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + if _, _, err := ctx.localVariableFor(nil, global, false); err == nil || !strings.Contains(err.Error(), "cannot use go:linkname") { t.Fatalf("localVariableFor error = %v", err) } assertLocalityPanic(t, "resolveLocality", func() { ctx.resolveLocality(name) }) diff --git a/cl/locality_test.go b/cl/locality_test.go index 694058bd90..6c0c6ea515 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -16,10 +16,6 @@ import ( ) func compileLocalitySource(t *testing.T, src string) (llssa.Program, string) { - return compileLocalitySourceWithNativeAnchor(t, src, true) -} - -func compileLocalitySourceWithNativeAnchor(t *testing.T, src string, nativeAnchor bool) (llssa.Program, string) { t.Helper() fset := token.NewFileSet() file, err := parser.ParseFile(fset, "locality.go", src, parser.ParseComments) @@ -36,11 +32,6 @@ func compileLocalitySourceWithNativeAnchor(t *testing.T, src string, nativeAncho prog := ssatest.NewProgramEx(t, nil, imp) prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) prog.SetRuntime(localityRuntimePackage()) - if nativeAnchor { - anchor := llssa.PkgRuntime + ".currentLocalContext" - prog.SetLocalityInfo(anchor, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) - prog.SetLocalStorage(anchor, llssa.LocalStorageNativeTLS) - } if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { t.Fatal(err) } @@ -76,15 +67,9 @@ func localityRuntimePackage() *types.Package { localContextName := types.NewTypeName(token.NoPos, pkg, "LocalContext", nil) localContext := types.NewNamed(localContextName, types.NewStruct([]*types.Var{blocks}, nil), nil) pkg.Scope().Insert(localContextName) - next := types.NewField(token.NoPos, pkg, "next", unsafePointer, false) - key := types.NewField(token.NoPos, pkg, "key", unsafePointer, false) - localBlockName := types.NewTypeName(token.NoPos, pkg, "localBlock", nil) - types.NewNamed(localBlockName, types.NewStruct([]*types.Var{next, key}, nil), nil) - pkg.Scope().Insert(localBlockName) - pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, "currentLocalContext", types.Typ[types.Uintptr])) localPackageParams := types.NewTuple( - types.NewParam(token.NoPos, pkg, "key", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, pkg, "cache", types.NewPointer(types.Typ[types.Uintptr])), types.NewParam(token.NoPos, pkg, "size", types.Typ[types.Uintptr]), types.NewParam(token.NoPos, pkg, "align", types.Typ[types.Uintptr]), ) @@ -94,7 +79,7 @@ func localityRuntimePackage() *types.Package { callback := types.NewSignatureType(nil, nil, nil, nil, nil, false) ensureParams := types.NewTuple( types.NewParam(token.NoPos, pkg, "state", types.NewPointer(types.Typ[types.Uint8])), - types.NewParam(token.NoPos, pkg, "failureKey", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, pkg, "failureCache", types.NewPointer(types.Typ[types.Uintptr])), types.NewParam(token.NoPos, pkg, "initialize", callback), ) pkg.Scope().Insert(types.NewFunc(token.NoPos, pkg, "EnsureLocalInitializer", types.NewSignatureType(nil, nil, nil, ensureParams, nil, false))) @@ -111,38 +96,29 @@ func localityRuntimePackage() *types.Package { return pkg } -func TestLocalPackageAccessorUsesRuntimeAnchorStorage(t *testing.T) { +func TestLocalPackageAccessorUsesDirectTLSCache(t *testing.T) { const src = `package locality //llgo:gls -var Pointer *int +var pointer *int -func value() *int { return Pointer } +func value() *int { return pointer } ` - for _, test := range []struct { - name string - nativeAnchor bool - declaration string - }{ - {"native TLS", true, "external thread_local global i64"}, - {"ordinary global", false, "external global i64"}, - } { - t.Run(test.name, func(t *testing.T) { - _, ir := compileLocalitySourceWithNativeAnchor(t, src, test.nativeAnchor) - anchor := `@"` + llssa.PkgRuntime + `.currentLocalContext" = ` + test.declaration - if !strings.Contains(ir, anchor) { - t.Fatalf("runtime context anchor declaration not found: %s\n%s", anchor, ir) - } - accessor := llvmFunction(t, ir, "example.com/locality.__llgo_local_block") - load := `load i64, ptr @"` + llssa.PkgRuntime + `.currentLocalContext"` - slow := `call ptr @"` + llssa.PkgRuntime + `.LocalPackage"` - if loadAt, slowAt := strings.Index(accessor, load), strings.Index(accessor, slow); loadAt < 0 || slowAt < 0 || loadAt >= slowAt { - t.Fatalf("accessor does not load the runtime anchor before its slow path:\n%s", accessor) - } - if got := strings.Count(accessor, "icmp "); got != 3 { - t.Fatalf("accessor comparisons = %d, want context/head/key checks:\n%s", got, accessor) - } - }) + _, ir := compileLocalitySource(t, src) + if !strings.Contains(ir, `@"example.com/locality.__llgo_local_cache" = thread_local global i64 0`) { + t.Fatalf("package direct cache not found:\n%s", ir) + } + accessor := llvmFunction(t, ir, "example.com/locality.__llgo_local_block") + cacheLoad := `load i64, ptr @"example.com/locality.__llgo_local_cache"` + slow := `call ptr @"` + llssa.PkgRuntime + `.LocalPackage"(ptr @"example.com/locality.__llgo_local_cache"` + if loadAt, slowAt := strings.Index(accessor, cacheLoad), strings.Index(accessor, slow); loadAt < 0 || slowAt < 0 || loadAt >= slowAt { + t.Fatalf("accessor does not load its direct cache before the cold path:\n%s", accessor) + } + if got := strings.Count(accessor, "icmp "); got != 1 { + t.Fatalf("accessor comparisons = %d, want one cache check:\n%s", got, accessor) + } + if strings.Contains(accessor, "currentLocalContext") { + t.Fatalf("accessor still depends on runtime context layout:\n%s", accessor) } } @@ -174,16 +150,16 @@ func scalar() int { return 42 } func pointer() *int { return &backing } //llgo:tls -var TLSScalar = scalar() +var tlsScalar = scalar() //llgo:tls -var TLSPointer = pointer() +var tlsPointer = pointer() //llgo:gls -var GLSScalar = scalar() +var glsScalar = scalar() //llgo:gls -var GLSPointer = pointer() +var glsPointer = pointer() func values() (int, *int, int, *int) { - return TLSScalar, TLSPointer, GLSScalar, GLSPointer + return tlsScalar, tlsPointer, glsScalar, glsPointer } `) @@ -191,10 +167,10 @@ func values() (int, *int, int, *int) { kind llssa.Locality storage llssa.LocalStorage }{ - "TLSScalar": {llssa.ThreadLocal, llssa.LocalStorageNativeTLS}, - "TLSPointer": {llssa.ThreadLocal, llssa.LocalStoragePackage}, - "GLSScalar": {llssa.GoroutineLocal, llssa.LocalStorageNativeTLS}, - "GLSPointer": {llssa.GoroutineLocal, llssa.LocalStoragePackage}, + "tlsScalar": {llssa.ThreadLocal, llssa.LocalStorageNativeTLS}, + "tlsPointer": {llssa.ThreadLocal, llssa.LocalStoragePackage}, + "glsScalar": {llssa.GoroutineLocal, llssa.LocalStorageNativeTLS}, + "glsPointer": {llssa.GoroutineLocal, llssa.LocalStoragePackage}, } for name, want := range checks { got, ok := prog.VariableLocality("example.com/locality." + name) @@ -202,18 +178,18 @@ func values() (int, *int, int, *int) { t.Fatalf("%s metadata = %+v, %v", name, got, ok) } } - for _, name := range []string{"TLSScalar", "GLSScalar"} { + for _, name := range []string{"tlsScalar", "glsScalar"} { if !strings.Contains(ir, `@"example.com/locality.`+name+`" = thread_local global i64`) { t.Fatalf("%s is not native TLS:\n%s", name, ir) } } - for _, name := range []string{"TLSPointer", "GLSPointer"} { + for _, name := range []string{"tlsPointer", "glsPointer"} { if strings.Contains(ir, `@"example.com/locality.`+name+`" = thread_local`) { t.Fatalf("%s retained a pointer-bearing TLS global:\n%s", name, ir) } } - if got := strings.Count(ir, `@"example.com/locality.__llgo_local_key" =`); got != 1 { - t.Fatalf("package block keys = %d, want 1:\n%s", got, ir) + if got := strings.Count(ir, `@"example.com/locality.__llgo_local_cache" = thread_local global i64 0`); got != 1 { + t.Fatalf("package block caches = %d, want 1:\n%s", got, ir) } if got := strings.Count(ir, `call ptr @"github.com/goplus/llgo/runtime/internal/runtime.LocalPackage"`); got != 1 { t.Fatalf("LocalPackage calls = %d, want one accessor definition:\n%s", got, ir) @@ -241,15 +217,15 @@ func TestLocalityDebugInfoOnlyUsesFixedGlobals(t *testing.T) { _, ir := compileLocalitySource(t, `package locality //llgo:tls -var Direct int +var direct int //llgo:gls -var Pointer *int +var pointer *int -func values() (int, *int) { return Direct, Pointer } +func values() (int, *int) { return direct, pointer } `) - direct := `@"example.com/locality.Direct" = thread_local global i64` + direct := `@"example.com/locality.direct" = thread_local global i64` start := strings.Index(ir, direct) if start < 0 { t.Fatalf("native TLS global not found:\n%s", ir) @@ -258,7 +234,7 @@ func values() (int, *int) { return Direct, Pointer } if end < 0 || !strings.Contains(ir[start:start+end], "!dbg") { t.Fatalf("native TLS global has no debug metadata:\n%s", ir[start:]) } - if strings.Contains(ir, `@"example.com/locality.Pointer" =`) { + if strings.Contains(ir, `@"example.com/locality.pointer" =`) { t.Fatalf("package-local pointer was emitted as a fixed debug global:\n%s", ir) } } @@ -268,12 +244,12 @@ func TestLocalityInitializersPreserveGoOrderPerKind(t *testing.T) { func mark(value int) int { return value } //llgo:tls -var T0 = mark(0) +var t0 = mark(0) //llgo:gls -var G0 = mark(1) +var g0 = mark(1) //llgo:tls -var T1 = mark(2) -func values() (int, int, int) { return T0, T1, G0 } +var t1 = mark(2) +func values() (int, int, int) { return t0, t1, g0 } `) tls := llvmFunction(t, ir, "example.com/locality.__llgo_tls_init") gls := llvmFunction(t, ir, "example.com/locality.__llgo_gls_init") @@ -297,8 +273,8 @@ func TestDirectInitializerStillRequiresFailureContext(t *testing.T) { prog, ir := compileLocalitySource(t, `package locality func value() int { return 1 } //llgo:tls -var Value = value() -func get() int { return Value } +var localValue = value() +func get() int { return localValue } `) if !prog.NeedsLocalContext() { t.Fatal("initializer failure storage did not enable a local context") @@ -306,7 +282,7 @@ func get() int { return Value } if strings.Contains(ir, `__llgo_local_block`) { t.Fatalf("pointer-free package unexpectedly has a value block:\n%s", ir) } - if !strings.Contains(ir, `@"example.com/locality.Value" = thread_local global i64`) || !strings.Contains(ir, `EnsureLocalInitializer`) { + if !strings.Contains(ir, `@"example.com/locality.localValue" = thread_local global i64`) || !strings.Contains(ir, `EnsureLocalInitializer`) { t.Fatalf("direct initializer lowering is incomplete:\n%s", ir) } } @@ -314,10 +290,10 @@ func get() int { return Value } func TestZeroValueDirectLocalsNeedNoContext(t *testing.T) { prog, ir := compileLocalitySource(t, `package locality //llgo:tls -var T int +var threadValue int //llgo:gls -var G uintptr -func values() (int, uintptr) { return T, G } +var goroutineValue uintptr +func values() (int, uintptr) { return threadValue, goroutineValue } `) if prog.NeedsLocalContext() { t.Fatal("pointer-free zero-value locals enabled a local context") @@ -330,10 +306,10 @@ func values() (int, uintptr) { return T, G } func TestExportedFunctionInstallsLocalContext(t *testing.T) { _, ir := compileLocalitySource(t, `package locality //llgo:gls -var Pointer *int +var pointer *int //export Exported func Exported(useLocal bool) *int { - if useLocal { return Pointer } + if useLocal { return pointer } return nil } `) @@ -355,9 +331,9 @@ func Exported(useLocal bool) *int { func TestExportedNativeTLSNeedsNoLocalContext(t *testing.T) { prog, ir := compileLocalitySource(t, `package locality //llgo:tls -var Scalar int +var scalar int //export Exported -func Exported() int { return Scalar } +func Exported() int { return scalar } `) if prog.NeedsLocalContext() { t.Fatal("zero-value native TLS enabled a local context") @@ -380,44 +356,32 @@ func assertTextOrder(t *testing.T, text string, wants ...string) { } } -func TestLocalityLinknameAliasesReuseCanonicalStorage(t *testing.T) { - prog, ir := compileLocalitySource(t, `package locality - -//llgo:gls -var Pointer *int -//go:linkname PointerAlias example.com/locality.Pointer +func TestLocalityRejectsLinknameAlias(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", `package locality //llgo:gls -var PointerAlias *int - -//llgo:tls -var Scalar int -//go:linkname ScalarAlias example.com/locality.Scalar -//llgo:tls -var ScalarAlias int - -func values() (*int, *int, int, int) { return Pointer, PointerAlias, Scalar, ScalarAlias } -`) - if strings.Contains(ir, "PointerAlias") || strings.Contains(ir, "ScalarAlias") { - t.Fatalf("linkname aliases received independent LLVM storage:\n%s", ir) +var target *int +//go:linkname alias example.com/locality.target +var alias *int +`, parser.ParseComments) + if err != nil { + t.Fatal(err) } - if got := strings.Count(ir, `@"example.com/locality.Scalar" = thread_local global i64`); got != 1 { - t.Fatalf("canonical scalar globals = %d, want 1:\n%s", got, ir) + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/locality", fset, []*ast.File{file}, info) + if err != nil { + t.Fatal(err) } - values := llvmFunction(t, ir, "example.com/locality.values") - if got := strings.Count(values, `call ptr @"example.com/locality.__llgo_local_block"()`); got != 1 { - t.Fatalf("alias package-base calls = %d, want 1:\n%s", got, values) + prog := ssatest.NewProgram(t, nil) + if err := ParsePkgSyntax(prog, fset, pkg, []*ast.File{file}); err != nil { + t.Fatal(err) } - for name, want := range map[string]llssa.LocalStorage{ - "example.com/locality.PointerAlias": llssa.LocalStoragePackage, - "example.com/locality.ScalarAlias": llssa.LocalStorageNativeTLS, - } { - if got, ok := prog.VariableLocality(name); !ok || got.LocalStorage != want { - t.Fatalf("alias metadata %s = %+v, %v; want storage %v", name, got, ok, want) - } + if err := prog.ValidateLocalities(pkg.Path()); err == nil || !strings.Contains(err.Error(), "cannot reference local variable") { + t.Fatalf("ValidateLocalities error = %v", err) } } -func TestLocalityCrossPackageAccessUsesDependencyStorage(t *testing.T) { +func TestLocalityCrossPackageAccessUsesFunctions(t *testing.T) { fset := token.NewFileSet() parse := func(name, source string) *ast.File { file, err := parser.ParseFile(fset, name, source, parser.ParseComments) @@ -429,13 +393,14 @@ func TestLocalityCrossPackageAccessUsesDependencyStorage(t *testing.T) { depFile := parse("dep.go", `package dep func initialScalar() int { return 1 } //llgo:tls -var Scalar = initialScalar() +var scalar = initialScalar() //llgo:gls -var Pointer *int +var pointer *int +func Values() (int, *int) { return scalar, pointer } `) rootFile := parse("root.go", `package root import "example.com/dep" -func Values() (int, *int) { return dep.Scalar, dep.Pointer } +func Values() (int, *int) { return dep.Values() } `) check := func(path string, files []*ast.File, imp types.Importer) (*types.Package, *types.Info) { info := newLocalityTypeInfo() @@ -484,44 +449,36 @@ func Values() (int, *int) { return dep.Scalar, dep.Pointer } t.Fatal(err) } ir := root.String() - if !strings.Contains(ir, `@"example.com/dep.Scalar" = external thread_local global i64`) { - t.Fatalf("root package did not reference dependency TLS storage:\n%s", ir) - } - if !strings.Contains(ir, `declare ptr @"example.com/dep.__llgo_local_block"()`) { - t.Fatalf("root package did not reference dependency block accessor:\n%s", ir) + if !strings.Contains(ir, `@"example.com/dep.Values"`) { + t.Fatalf("root package did not call the dependency function:\n%s", ir) } - if !strings.Contains(ir, `declare void @"example.com/dep.__llgo_tls_init$ensure"()`) { - t.Fatalf("root package did not reference dependency initializer guard:\n%s", ir) - } - if strings.Contains(ir, `define ptr @"example.com/dep.__llgo_local_block"()`) { - t.Fatalf("root package redefined dependency block accessor:\n%s", ir) + for _, symbol := range []string{"example.com/dep.scalar", "example.com/dep.pointer", "example.com/dep.__llgo_"} { + if strings.Contains(ir, symbol) { + t.Fatalf("root package referenced dependency locality storage %q:\n%s", symbol, ir) + } } } -func TestPrepareRejectsLocalAliasInitializer(t *testing.T) { +func TestParseRejectsLocalAliasInitializer(t *testing.T) { fset := token.NewFileSet() file, err := parser.ParseFile(fset, "locality.go", `package locality //llgo:tls -var Target int -//go:linkname Alias example.com/locality.Target +var target int +//go:linkname alias example.com/locality.target //llgo:tls -var Alias = 1 +var alias = 1 `, parser.ParseComments) if err != nil { t.Fatal(err) } files := []*ast.File{file} - info := newLocalityTypeInfo() - pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, newLocalityTypeInfo()) if err != nil { t.Fatal(err) } prog := llssa.NewProgram(nil) - if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { - t.Fatal(err) - } - if err := PrepareLocalVariables(prog, fset, pkg, info, files); err == nil || !strings.Contains(err.Error(), "linkname alias") { - t.Fatalf("PrepareLocalVariables error = %v", err) + if err := ParsePkgSyntax(prog, fset, pkg, files); err == nil || !strings.Contains(err.Error(), "cannot apply to a //go:linkname variable") { + t.Fatalf("ParsePkgSyntax error = %v", err) } } @@ -565,15 +522,15 @@ var value = initialValue() wantError: "inconsistent initializer metadata", }, { - name: "linkname locality mismatch", + name: "linkname locality", src: `package locality //llgo:tls -var Target int -//go:linkname Alias example.com/locality.Target +var target int +//go:linkname alias example.com/locality.target //llgo:gls -var Alias int +var alias int `, - wantError: "uses //llgo:gls", + wantError: "cannot apply to a //go:linkname variable", }, } @@ -606,10 +563,9 @@ var Alias int } } -func TestPrepareRejectsLocalAliasWithoutLocalTarget(t *testing.T) { +func TestParseRejectsExportedLocalVariable(t *testing.T) { fset := token.NewFileSet() file, err := parser.ParseFile(fset, "locality.go", `package locality -//go:linkname Value C.value //llgo:tls var Value int `, parser.ParseComments) @@ -617,17 +573,13 @@ var Value int t.Fatal(err) } files := []*ast.File{file} - info := newLocalityTypeInfo() - pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, newLocalityTypeInfo()) if err != nil { t.Fatal(err) } prog := ssatest.NewProgram(t, nil) - if err := ParsePkgSyntax(prog, fset, pkg, files); err != nil { - t.Fatal(err) - } - if err := PrepareLocalVariables(prog, fset, pkg, info, files); err == nil || !strings.Contains(err.Error(), "is not a local variable") { - t.Fatalf("PrepareLocalVariables error = %v", err) + if err := ParsePkgSyntax(prog, fset, pkg, files); err == nil || !strings.Contains(err.Error(), "requires an unexported package variable") { + t.Fatalf("ParsePkgSyntax error = %v", err) } } @@ -663,7 +615,7 @@ func TestPrepareLocalVariablesRejectsInvalidMetadata(t *testing.T) { t.Fatalf("PrepareLocalVariables error = %v", err) } }) - t.Run("linkname cycle", func(t *testing.T) { + t.Run("local linkname", func(t *testing.T) { prog := ssatest.NewProgram(t, nil) pkg := types.NewPackage("example.com/cycle", "cycle") first := llssa.FullName(pkg, "First") @@ -672,7 +624,7 @@ func TestPrepareLocalVariablesRejectsInvalidMetadata(t *testing.T) { prog.SetLocalityInfo(first, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) prog.SetLinkname(first, second) prog.SetLinkname(second, first) - if err := PrepareLocalVariables(prog, nil, pkg, &types.Info{}, nil); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + if err := PrepareLocalVariables(prog, nil, pkg, &types.Info{}, nil); err == nil || !strings.Contains(err.Error(), "cannot use go:linkname") { t.Fatalf("PrepareLocalVariables error = %v", err) } }) @@ -716,10 +668,10 @@ func TestNamedPointerLocalUsesPackageStorage(t *testing.T) { type Handle struct { Pointer *int } func makeHandle() Handle { return Handle{} } //llgo:tls -var Value = makeHandle() -func get() Handle { return Value } +var value = makeHandle() +func get() Handle { return value } `) - info, ok := prog.VariableLocality("example.com/locality.Value") + info, ok := prog.VariableLocality("example.com/locality.value") if !ok || info.LocalStorage != llssa.LocalStoragePackage { t.Fatalf("named pointer metadata = %+v, %v", info, ok) } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 6584ad3c87..c0a20adb8c 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -1117,24 +1117,24 @@ func Invalid() {} } } -func TestDoReportsLocalityAliasInitializer(t *testing.T) { +func TestDoRejectsLocalityLinkname(t *testing.T) { file := filepath.Join(t.TempDir(), "invalid_locality_alias.go") if err := os.WriteFile(file, []byte(`package invalidlocalityalias import _ "unsafe" //llgo:tls -var Target int +var target int -//go:linkname Alias example.com/target.Value +//go:linkname alias example.com/target.value //llgo:tls -var Alias = 1 +var alias = 1 `), 0o644); err != nil { t.Fatal(err) } conf := NewDefaultConf(ModeGen) - if _, err := Do([]string{file}, conf); err == nil || !strings.Contains(err.Error(), "linkname alias") { - t.Fatalf("Do error = %v, want locality alias initializer diagnostic", err) + if _, err := Do([]string{file}, conf); err == nil || !strings.Contains(err.Error(), "cannot apply to a //go:linkname variable") { + t.Fatalf("Do error = %v, want locality linkname diagnostic", err) } } diff --git a/internal/locality/layout/layout.go b/internal/locality/layout/layout.go index 0589c39f1f..2c8b8fbdfb 100644 --- a/internal/locality/layout/layout.go +++ b/internal/locality/layout/layout.go @@ -184,8 +184,8 @@ func (p Package) Initializers(kind locality.Kind) []Initializer { // BlockName returns the shared package-block accessor symbol. func BlockName(path string) string { return qualify(path, "__llgo_local_block") } -// BlockKeyName returns the shared package-block descriptor symbol. -func BlockKeyName(path string) string { return qualify(path, "__llgo_local_key") } +// BlockCacheName returns the package block's owner-local direct-cache symbol. +func BlockCacheName(path string) string { return qualify(path, "__llgo_local_cache") } // InitName returns the package/kind initializer dispatcher symbol. func InitName(path string, kind locality.Kind) string { @@ -198,8 +198,10 @@ func EnsureName(path string, kind locality.Kind) string { return InitName(path, // GuardName returns the package/kind native TLS state symbol. func GuardName(path string, kind locality.Kind) string { return InitName(path, kind) + "$guard" } -// FailureKeyName returns the package/kind initializer failure key symbol. -func FailureKeyName(path string, kind locality.Kind) string { return InitName(path, kind) + "$failure" } +// FailureCacheName returns the package/kind initializer failure-cache symbol. +func FailureCacheName(path string, kind locality.Kind) string { + return InitName(path, kind) + "$failure_cache" +} func qualify(path, name string) string { if path == "" { diff --git a/internal/locality/layout/layout_test.go b/internal/locality/layout/layout_test.go index 3aabeb2b79..af2527e3af 100644 --- a/internal/locality/layout/layout_test.go +++ b/internal/locality/layout/layout_test.go @@ -101,7 +101,7 @@ func TestNames(t *testing.T) { if got := BlockName("example.com/p"); got != "example.com/p.__llgo_local_block" { t.Fatal(got) } - if got := BlockKeyName("example.com/p"); got != "example.com/p.__llgo_local_key" { + if got := BlockCacheName("example.com/p"); got != "example.com/p.__llgo_local_cache" { t.Fatal(got) } if got := InitName("example.com/p", locality.Thread); got != "example.com/p.__llgo_tls_init" { @@ -113,7 +113,7 @@ func TestNames(t *testing.T) { if got := GuardName("", locality.Thread); got != "__llgo_tls_init$guard" { t.Fatal(got) } - if got := FailureKeyName("", locality.Goroutine); got != "__llgo_gls_init$failure" { + if got := FailureCacheName("", locality.Goroutine); got != "__llgo_gls_init$failure_cache" { t.Fatal(got) } } diff --git a/internal/locality/locality_test.go b/internal/locality/locality_test.go index 4a8ac7d44b..f46bcccc2d 100644 --- a/internal/locality/locality_test.go +++ b/internal/locality/locality_test.go @@ -121,6 +121,16 @@ func TestScanPackageVarBranches(t *testing.T) { decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//llgo:tls"), Specs: []ast.Spec{&ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("_")}}}}, want: "blank identifier", }, + { + name: "exported name", + decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//llgo:gls"), Specs: []ast.Spec{&ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("Value")}}}}, + want: "requires an unexported package variable", + }, + { + name: "linkname", + decl: &ast.GenDecl{Tok: token.VAR, Doc: comment("//go:linkname value example.com/p.value\n//llgo:tls"), Specs: []ast.Spec{&ast.ValueSpec{Names: []*ast.Ident{ast.NewIdent("value")}}}}, + want: "cannot apply to a //go:linkname variable", + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -285,7 +295,7 @@ func TestPrepareIsIdempotentAcrossPrograms(t *testing.T) { fset, file := parseFile(t, `package p func makeValue() *int { value := 42; return &value } //llgo:tls -var Value = makeValue() +var value = makeValue() `) files := []*ast.File{file} info := newTypeInfo() @@ -308,9 +318,9 @@ var Value = makeValue() } declCount := len(file.Decls) scopeCount := len(pkg.Scope().Names()) - initName := prepared["Value"].InitFunc - if initName != "example.com/p.__llgo_local_init_0" || prepared["Value"].InitOrder != 1 { - t.Fatalf("prepared metadata = %+v", prepared["Value"]) + initName := prepared["value"].InitFunc + if initName != "example.com/p.__llgo_local_init_0" || prepared["value"].InitOrder != 1 { + t.Fatalf("prepared metadata = %+v", prepared["value"]) } again, err := Prepare(fset, pkg.Path(), pkg, info, files, prepared) @@ -324,8 +334,8 @@ var Value = makeValue() if len(file.Decls) != declCount || len(pkg.Scope().Names()) != scopeCount { t.Fatalf("repeated Prepare changed syntax/scope: decls=%d/%d scope=%d/%d", len(file.Decls), declCount, len(pkg.Scope().Names()), scopeCount) } - if again["Value"] != prepared["Value"] || reused["Value"] != prepared["Value"] { - t.Fatalf("repeated metadata = %+v, reused = %+v, want %+v", again["Value"], reused["Value"], prepared["Value"]) + if again["value"] != prepared["value"] || reused["value"] != prepared["value"] { + t.Fatalf("repeated metadata = %+v, reused = %+v, want %+v", again["value"], reused["value"], prepared["value"]) } } diff --git a/internal/locality/scan.go b/internal/locality/scan.go index b458a81795..5032133baa 100644 --- a/internal/locality/scan.go +++ b/internal/locality/scan.go @@ -62,10 +62,16 @@ func ScanPackageVar(fset *token.FileSet, decl *ast.GenDecl) ([]Variable, error) if hasDirective(decl.Doc, "go:embed") || hasDirective(spec.Doc, "go:embed") { return nil, errorAt(fset, spec.Pos(), "%s and //go:embed cannot apply to the same variable declaration", Directive(kind)) } + if hasDirective(decl.Doc, "go:linkname") || hasDirective(spec.Doc, "go:linkname") { + return nil, errorAt(fset, spec.Pos(), "%s cannot apply to a //go:linkname variable", Directive(kind)) + } for _, ident := range spec.Names { if ident.Name == "_" { return nil, errorAt(fset, ident.Pos(), "locality directive cannot apply to the blank identifier") } + if ast.IsExported(ident.Name) { + return nil, errorAt(fset, ident.Pos(), "%s requires an unexported package variable", Directive(kind)) + } ret = append(ret, Variable{ Name: ident.Name, Info: Info{Locality: kind, HasInitializer: len(spec.Values) != 0}, diff --git a/runtime/internal/runtime/local_context.go b/runtime/internal/runtime/local_context.go index 9ce43dcdac..5425984bb6 100644 --- a/runtime/internal/runtime/local_context.go +++ b/runtime/internal/runtime/local_context.go @@ -24,16 +24,15 @@ import "unsafe" // one-thread-per-goroutine backend maps both logical locality kinds to this one // physical package store. type LocalContext struct { - // blocks points at the payload of the most recently used package block. - // Keeping the aligned payload at the head makes the common lookup return it - // directly; the block header is stored immediately before the payload. + // blocks points at the payload of the most recently allocated local block. + // The list keeps every block reachable from the outer Go entry stack frame. blocks unsafe.Pointer } type localBlock struct { // next points at the next block's payload, not its header. - next unsafe.Pointer - key unsafe.Pointer + next unsafe.Pointer + cacheSlot *uintptr } func EnterLocalContext(ctx *LocalContext) uintptr { @@ -78,57 +77,40 @@ func releaseLocalBlocks(ctx *LocalContext) { next := block.next // Do not free block here: an address of a local variable may outlive its // owner. Breaking the links lets the GC retain only escaped blocks. + *block.cacheSlot = 0 block.next = nil + block.cacheSlot = nil data = next } } -// LocalPackage returns stable, zeroed storage for one package in the current -// physical owner. Repeated access to the most recently used package takes the -// head fast path; other accesses move the matching block to the front. -func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { - ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) - if ctx != nil { - firstData := ctx.blocks - if firstData != nil && localBlockHeader(firstData).key == key { - return firstData - } - } - return localPackageSlow(ctx, key, size, align) -} - +// LocalPackage creates stable, zeroed storage for one generated cache slot in +// the current physical owner. Generated accessors load the slot directly after +// first touch; the block list is retained only as a GC root and teardown list. +// //go:noinline -func localPackageSlow(ctx *LocalContext, key unsafe.Pointer, size, align uintptr) unsafe.Pointer { +func LocalPackage(cacheSlot *uintptr, size, align uintptr) unsafe.Pointer { + ctx := (*LocalContext)(unsafe.Pointer(currentLocalContext)) if ctx == nil { panic("runtime: local variable accessed outside a Go entry context") } - if key == nil { - panic("runtime: nil local package key") + if cacheSlot == nil { + panic("runtime: nil local cache slot") + } + if data := unsafe.Pointer(*cacheSlot); data != nil { + return data } if align == 0 || align&(align-1) != 0 { panic("runtime: invalid local package alignment") } - firstData := ctx.blocks - var previous *localBlock - for data := firstData; data != nil; { - block := localBlockHeader(data) - next := block.next - if block.key == key { - previous.next = next - block.next = firstData - ctx.blocks = data - return data - } - previous = block - data = next - } - data := newLocalBlock(key, size, align) - localBlockHeader(data).next = firstData + data := newLocalBlock(cacheSlot, size, align) + localBlockHeader(data).next = ctx.blocks ctx.blocks = data + *cacheSlot = uintptr(data) return data } -func newLocalBlock(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { +func newLocalBlock(cacheSlot *uintptr, size, align uintptr) unsafe.Pointer { header := unsafe.Sizeof(localBlock{}) padding := align - 1 if size == 0 { @@ -143,7 +125,7 @@ func newLocalBlock(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { } data := unsafe.Pointer((uintptr(allocation) + header + padding) &^ padding) block := localBlockHeader(data) - block.key = key + block.cacheSlot = cacheSlot return data } diff --git a/runtime/internal/runtime/local_context_stub.go b/runtime/internal/runtime/local_context_stub.go index ebd713e575..4dd029e4ef 100644 --- a/runtime/internal/runtime/local_context_stub.go +++ b/runtime/internal/runtime/local_context_stub.go @@ -30,6 +30,13 @@ func LeaveLocalContext(ctx *LocalContext, previous uintptr) {} func leaveCurrentLocalContext() {} -func LocalPackage(key unsafe.Pointer, size, align uintptr) unsafe.Pointer { - return AllocZ(size) +func LocalPackage(cacheSlot *uintptr, size, align uintptr) unsafe.Pointer { + if cacheSlot != nil && *cacheSlot != 0 { + return unsafe.Pointer(*cacheSlot) + } + data := AllocZ(size) + if cacheSlot != nil { + *cacheSlot = uintptr(data) + } + return data } diff --git a/runtime/internal/runtime/local_initializer.go b/runtime/internal/runtime/local_initializer.go index 58cb8a7fc0..71fa122eda 100644 --- a/runtime/internal/runtime/local_initializer.go +++ b/runtime/internal/runtime/local_initializer.go @@ -28,14 +28,14 @@ const ( // EnsureLocalInitializer executes one package/locality dispatcher at most once // in the current owner. Recursive access observes partial initialization; a // recovered failure remains failed and re-panics on every later access. -func EnsureLocalInitializer(state *uint8, failureKey unsafe.Pointer, initialize func()) { +func EnsureLocalInitializer(state *uint8, failureCache *uintptr, initialize func()) { switch *state { case localInitReady: return case localInitInitializing: return case localInitFailed: - panic(*localInitializerFailure(failureKey)) + panic(*localInitializerFailure(failureCache)) case localInitUninitialized: default: panic("runtime: invalid local initializer state") @@ -47,7 +47,7 @@ func EnsureLocalInitializer(state *uint8, failureKey unsafe.Pointer, initialize return } value := recover() - *localInitializerFailure(failureKey) = value + *localInitializerFailure(failureCache) = value *state = localInitFailed panic(value) }() @@ -56,7 +56,7 @@ func EnsureLocalInitializer(state *uint8, failureKey unsafe.Pointer, initialize *state = localInitReady } -func localInitializerFailure(key unsafe.Pointer) *any { +func localInitializerFailure(cache *uintptr) *any { var value any - return (*any)(LocalPackage(key, unsafe.Sizeof(value), unsafe.Alignof(value))) + return (*any)(LocalPackage(cache, unsafe.Sizeof(value), unsafe.Alignof(value))) } diff --git a/ssa/local_context.go b/ssa/local_context.go index cec3e2878a..184b2b5ec4 100644 --- a/ssa/local_context.go +++ b/ssa/local_context.go @@ -21,11 +21,7 @@ import ( "go/types" ) -const ( - runtimeCurrentLocalContext = "currentLocalContext" - runtimeLocalContext = "LocalContext" - runtimeLocalBlock = "localBlock" -) +const runtimeLocalContext = "LocalContext" // EnterLocalContext creates the stack root used by TLS/GLS locality blocks and // installs it for the current outermost Go entry. previous is nonzero only for @@ -46,64 +42,23 @@ func (b Builder) LeaveLocalContext(ctx, previous Expr) { b.Call(b.Pkg.rtFunc("LeaveLocalContext"), ctx, previous) } -// BuildLocalPackageAccessor builds an always-hot package block lookup around -// LocalPackage. Runtime Go types and locality metadata are the ABI: they supply -// the context/header layouts and whether the context anchor is native TLS or an -// ordinary global. The generated code therefore does not duplicate target -// selection or byte offsets. -func (p Function) BuildLocalPackageAccessor(key, size, align Expr) { +// BuildLocalPackageAccessor builds an always-hot package block lookup around a +// generated owner-local cache slot. The runtime owns allocation and rooting; +// the compiler needs no LocalContext or block-header layout knowledge. +func (p Function) BuildLocalPackageAccessor(cache, size, align Expr) { prog := p.Prog - runtimePkg := prog.runtime() - contextType := runtimePkg.Scope().Lookup(runtimeLocalContext).Type() - blockType := runtimePkg.Scope().Lookup(runtimeLocalBlock).Type() - _, contextBlocks, _ := types.LookupFieldOrMethod(contextType, true, runtimePkg, "blocks") - _, blockKey, _ := types.LookupFieldOrMethod(blockType, true, runtimePkg, "key") - - b := p.MakeBody(5) - checkHead := p.Block(1) - checkKey := p.Block(2) - hit := p.Block(3) - slow := p.Block(4) - key = b.Convert(prog.VoidPtr(), key) - - current := b.Load(p.Pkg.runtimeGlobal(runtimeCurrentLocalContext).Expr) - hasContext := b.BinOp(token.NEQ, current, prog.IntVal(0, prog.Uintptr())) - b.If(hasContext, checkHead, slow) - - b.SetBlock(checkHead) - context := b.Convert(prog.Pointer(prog.Type(contextType, InGo)), current) - head := b.Load(b.FieldAddr(context, contextBlocks[0])) - hasHead := b.BinOp(token.NEQ, head, prog.Nil(head.Type)) - b.If(hasHead, checkKey, slow) - - b.SetBlock(checkKey) - headerAddress := b.BinOp( - token.SUB, - b.Convert(prog.Uintptr(), head), - prog.IntVal(prog.SizeOf(prog.Type(blockType, InGo)), prog.Uintptr()), - ) - header := b.Convert(prog.Pointer(prog.Type(blockType, InGo)), headerAddress) - foundKey := b.Load(b.FieldAddr(header, blockKey[0])) - b.If(b.BinOp(token.EQL, foundKey, key), hit, slow) + b := p.MakeBody(3) + hit := p.Block(1) + slow := p.Block(2) + cached := b.Load(cache) + b.If(b.BinOp(token.NEQ, cached, prog.IntVal(0, prog.Uintptr())), hit, slow) b.SetBlock(hit) result := p.raw.Type.(*types.Signature).Results().At(0).Type() - b.Return(b.Convert(prog.rawType(result), head)) + b.Return(b.Convert(prog.rawType(result), cached)) b.SetBlock(slow) - raw := b.Call(p.Pkg.rtFunc("LocalPackage"), key, size, align) + raw := b.Call(p.Pkg.rtFunc("LocalPackage"), cache, size, align) b.Return(b.Convert(prog.rawType(result), raw)) b.EndBuild() } - -func (p Package) runtimeGlobal(name string) Global { - p.NeedRuntime = true - runtimePkg := p.Prog.runtime() - variable := runtimePkg.Scope().Lookup(name).(*types.Var) - fullName := FullName(runtimePkg, name) - typ := types.NewPointer(variable.Type()) - if locality, ok := p.Prog.VariableLocality(fullName); ok && locality.LocalStorage == LocalStorageNativeTLS { - return p.NewThreadLocalVar(fullName, typ, InGo) - } - return p.NewVar(fullName, typ, InGo) -} diff --git a/ssa/local_context_test.go b/ssa/local_context_test.go index 9517b13a9f..624696e635 100644 --- a/ssa/local_context_test.go +++ b/ssa/local_context_test.go @@ -24,50 +24,32 @@ import ( ) func TestBuildLocalPackageAccessor(t *testing.T) { - for _, test := range []struct { - name string - nativeAnchor bool - declaration string - }{ - {"native TLS anchor", true, "external thread_local global i64"}, - {"ordinary anchor", false, "external global i64"}, - } { - t.Run(test.name, func(t *testing.T) { - prog := NewProgram(nil) - prog.SetRuntime(localContextTestRuntime()) - anchorName := PkgRuntime + "." + runtimeCurrentLocalContext - if test.nativeAnchor { - prog.SetLocalityInfo(anchorName, LocalityInfo{Locality: ThreadLocal}) - prog.SetLocalStorage(anchorName, LocalStorageNativeTLS) - } - - pkg := prog.NewPackage("accessor", "example.com/accessor") - key := pkg.NewVar("example.com/accessor.key", types.NewPointer(types.Typ[types.Uint8]), InGo) - key.InitNil() - field := types.NewField(token.NoPos, nil, "pointer", types.NewPointer(types.Typ[types.Int]), false) - block := types.NewStruct([]*types.Var{field}, nil) - blockType := prog.Type(block, InGo) - result := types.NewPointer(block) - results := types.NewTuple(types.NewVar(token.NoPos, nil, "", result)) - accessor := pkg.NewFunc("example.com/accessor.block", types.NewSignatureType(nil, nil, nil, nil, results, false), InGo) - accessor.BuildLocalPackageAccessor( - key.Expr, - prog.IntVal(prog.SizeOf(blockType), prog.Uintptr()), - prog.IntVal(prog.AlignOf(blockType), prog.Uintptr()), - ) + prog := NewProgram(nil) + prog.SetRuntime(localContextTestRuntime()) + pkg := prog.NewPackage("accessor", "example.com/accessor") + cache := pkg.NewThreadLocalVar("example.com/accessor.cache", types.NewPointer(types.Typ[types.Uintptr]), InGo) + cache.InitNil() + field := types.NewField(token.NoPos, nil, "pointer", types.NewPointer(types.Typ[types.Int]), false) + block := types.NewStruct([]*types.Var{field}, nil) + blockType := prog.Type(block, InGo) + result := types.NewPointer(block) + results := types.NewTuple(types.NewVar(token.NoPos, nil, "", result)) + accessor := pkg.NewFunc("example.com/accessor.block", types.NewSignatureType(nil, nil, nil, nil, results, false), InGo) + accessor.BuildLocalPackageAccessor( + cache.Expr, + prog.IntVal(prog.SizeOf(blockType), prog.Uintptr()), + prog.IntVal(prog.AlignOf(blockType), prog.Uintptr()), + ) - ir := pkg.String() - anchor := `@"` + anchorName + `" = ` + test.declaration - if !strings.Contains(ir, anchor) { - t.Fatalf("context anchor declaration not found: %s\n%s", anchor, ir) - } - if got := strings.Count(ir, "icmp "); got != 3 { - t.Fatalf("accessor comparisons = %d, want context/head/key checks:\n%s", got, ir) - } - if !strings.Contains(ir, `call ptr @"`+PkgRuntime+`.LocalPackage"`) { - t.Fatalf("accessor has no LocalPackage slow path:\n%s", ir) - } - }) + ir := pkg.String() + if !strings.Contains(ir, `@"example.com/accessor.cache" = thread_local global i64 0`) { + t.Fatalf("direct cache definition not found:\n%s", ir) + } + if got := strings.Count(ir, "icmp "); got != 1 { + t.Fatalf("accessor comparisons = %d, want one cache check:\n%s", got, ir) + } + if !strings.Contains(ir, `call ptr @"`+PkgRuntime+`.LocalPackage"(ptr @"example.com/accessor.cache"`) { + t.Fatalf("accessor has no cache-backed LocalPackage slow path:\n%s", ir) } } @@ -75,22 +57,8 @@ func localContextTestRuntime() *types.Package { pkg := types.NewPackage(PkgRuntime, "runtime") unsafePointer := types.Typ[types.UnsafePointer] - contextName := types.NewTypeName(token.NoPos, pkg, runtimeLocalContext, nil) - contextFields := []*types.Var{types.NewField(token.NoPos, pkg, "blocks", unsafePointer, false)} - types.NewNamed(contextName, types.NewStruct(contextFields, nil), nil) - pkg.Scope().Insert(contextName) - - blockName := types.NewTypeName(token.NoPos, pkg, runtimeLocalBlock, nil) - blockFields := []*types.Var{ - types.NewField(token.NoPos, pkg, "next", unsafePointer, false), - types.NewField(token.NoPos, pkg, "key", unsafePointer, false), - } - types.NewNamed(blockName, types.NewStruct(blockFields, nil), nil) - pkg.Scope().Insert(blockName) - pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, runtimeCurrentLocalContext, types.Typ[types.Uintptr])) - params := types.NewTuple( - types.NewVar(token.NoPos, pkg, "key", unsafePointer), + types.NewVar(token.NoPos, pkg, "cache", types.NewPointer(types.Typ[types.Uintptr])), types.NewVar(token.NoPos, pkg, "size", types.Typ[types.Uintptr]), types.NewVar(token.NoPos, pkg, "align", types.Typ[types.Uintptr]), ) diff --git a/ssa/locality.go b/ssa/locality.go index 4cfbb68fa5..a8e9a62a7c 100644 --- a/ssa/locality.go +++ b/ssa/locality.go @@ -86,8 +86,8 @@ func (p Program) VariableLocality(name string) (VariableLocality, bool) { return info, ok } -// ResolveLocality follows linkname aliases and returns the canonical declaration -// name together with its merged locality metadata. +// ResolveLocality returns locality metadata for one declaration. Locality +// variables cannot participate in go:linkname alias chains. func (p Program) ResolveLocality(name string) (string, VariableLocality, bool, error) { lookup := func(name string) (VariableLocality, bool) { p.localities.mu.RLock() @@ -112,27 +112,17 @@ func resolveLocality(lookup func(string) (VariableLocality, bool), linkname func seen[current] = true target, hasLink := linkname(current) target = strings.TrimPrefix(target, "go:") - if !hasLink || target == "" || target == current { + if !hasLink || target == "" { return current, result, ok, nil } - targetInfo, exists := lookup(target) - if exists && targetInfo.Locality != locality.None { - switch { - case result.Locality == locality.None: - result = targetInfo - ok = true - case result.Locality != targetInfo.Locality: - return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s uses %s but target %s uses %s", name, locality.Directive(result.Locality), target, locality.Directive(targetInfo.Locality)) - case hasInitialization(result.Info) && hasInitialization(targetInfo.Info) && result.Info != targetInfo.Info: - return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s and target %s have incompatible local initializers", name, target) - case !hasInitialization(result.Info): - result.Info = targetInfo.Info - } - if result.LocalStorage == LocalStorageUnknown { - result.LocalStorage = targetInfo.LocalStorage - } else if targetInfo.LocalStorage != LocalStorageUnknown && result.LocalStorage != targetInfo.LocalStorage { - return "", VariableLocality{}, false, fmt.Errorf("linkname alias %s and target %s have incompatible local storage", name, target) - } + if currentInfo, exists := lookup(current); exists && currentInfo.Locality != locality.None { + return "", VariableLocality{}, false, fmt.Errorf("local variable %s cannot use go:linkname", current) + } + if targetInfo, exists := lookup(target); exists && targetInfo.Locality != locality.None { + return "", VariableLocality{}, false, fmt.Errorf("go:linkname alias %s cannot reference local variable %s", name, target) + } + if target == current { + return current, result, ok, nil } current = target } @@ -145,13 +135,32 @@ func hasInitialization(info locality.Info) bool { func (p Program) ValidateLocalities(pkgPath string) error { prefix := pkgPath + "." p.localities.mu.RLock() - names := make([]string, 0) - for name := range p.localities.entries { + nameSet := make(map[string]bool) + localNames := make(map[string]bool) + for name, info := range p.localities.entries { + if info.Locality != locality.None { + localNames[name] = true + } if strings.HasPrefix(name, prefix) { - names = append(names, name) + nameSet[name] = true } } p.localities.mu.RUnlock() + p.linknameMu.RLock() + links := make(map[string]string, len(p.linkname)) + for name, target := range p.linkname { + links[name] = strings.TrimPrefix(target, "go:") + } + p.linknameMu.RUnlock() + for name := range links { + if strings.HasPrefix(name, prefix) && linknameReachesLocal(name, links, localNames) { + nameSet[name] = true + } + } + names := make([]string, 0, len(nameSet)) + for name := range nameSet { + names = append(names, name) + } sort.Strings(names) for _, name := range names { if _, _, _, err := p.ResolveLocality(name); err != nil { @@ -161,6 +170,18 @@ func (p Program) ValidateLocalities(pkgPath string) error { return nil } +func linknameReachesLocal(name string, links map[string]string, localNames map[string]bool) bool { + seen := make(map[string]bool) + for name != "" && !seen[name] { + if localNames[name] { + return true + } + seen[name] = true + name = links[name] + } + return false +} + func (p Program) PackageSyntaxParsed(pkg *types.Package) bool { p.localities.mu.RLock() _, ok := p.localities.parsedPackages[pkg] diff --git a/ssa/locality_test.go b/ssa/locality_test.go index 991c80808d..e931797443 100644 --- a/ssa/locality_test.go +++ b/ssa/locality_test.go @@ -86,74 +86,50 @@ func TestNeedsLocalContext(t *testing.T) { } } -func TestResolveLinknameLocality(t *testing.T) { +func TestRejectsLinknameLocality(t *testing.T) { prog := NewProgram(nil) - target := "example.com/target.Value" - alias := "example.com/alias.Value" + target := "example.com/target.value" + alias := "example.com/alias.value" prog.SetLocalityInfo(target, LocalityInfo{Locality: ThreadLocal, HasInitializer: true, InitFunc: "example.com/target.initValue", InitOrder: 1}) prog.SetLocalStorage(target, LocalStoragePackage) - prog.SetLinkname(alias, target) - - _, got, ok, err := prog.ResolveLocality(alias) - if err != nil { - t.Fatal(err) - } - if !ok || got.Locality != ThreadLocal || got.LocalStorage != LocalStoragePackage || got.InitFunc != "example.com/target.initValue" || got.InitOrder != 1 { - t.Fatalf("ResolveLocality(%q) = %+v, %v", alias, got, ok) + if canonical, got, ok, err := prog.ResolveLocality(target); err != nil || canonical != target || !ok || got.LocalStorage != LocalStoragePackage { + t.Fatalf("direct ResolveLocality(%q) = %q, %+v, %v, %v", target, canonical, got, ok, err) } - if err := prog.ValidateLocalities("example.com/alias"); err != nil { - t.Fatal(err) - } - sameKind := "example.com/alias.SameKind" - prog.SetLinkname(sameKind, target) - prog.SetLocalityInfo(sameKind, LocalityInfo{Locality: ThreadLocal}) - if _, got, ok, err := prog.ResolveLocality(sameKind); err != nil || !ok || got.InitFunc != "example.com/target.initValue" { - t.Fatalf("same-kind ResolveLocality(%q) = %+v, %v", sameKind, got, ok) - } - - incompatible := "example.com/alias.Incompatible" - prog.SetLinkname(incompatible, target) - prog.SetLocalityInfo(incompatible, LocalityInfo{Locality: ThreadLocal, HasInitializer: true, InitFunc: "example.com/alias.initValue", InitOrder: 1}) - if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "incompatible local initializers") { - t.Fatalf("initializer mismatch error = %v", err) - } - targetDecl, _ := prog.VariableLocality(target) - prog.SetLocalityInfo(incompatible, targetDecl.Info) - storageMismatch := "example.com/alias.StorageMismatch" - prog.SetLinkname(storageMismatch, target) - prog.SetLocalityInfo(storageMismatch, LocalityInfo{Locality: ThreadLocal}) - prog.SetLocalStorage(storageMismatch, LocalStorageNativeTLS) - if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "incompatible local storage") { - t.Fatalf("storage mismatch error = %v", err) - } - prog.SetLocalStorage(storageMismatch, LocalStoragePackage) + prog.SetLinkname(alias, target) + if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "cannot reference local variable") { + t.Fatalf("alias-to-local error = %v", err) + } - prog.SetLocalityInfo(alias, LocalityInfo{Locality: GoroutineLocal}) - if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "uses //llgo:gls") { - t.Fatalf("locality mismatch error = %v", err) + localAlias := "example.com/alias.local" + prog.SetLocalityInfo(localAlias, LocalityInfo{Locality: GoroutineLocal}) + prog.SetLinkname(localAlias, "example.com/target.ordinary") + if err := prog.ValidateLocalities("example.com/alias"); err == nil || !strings.Contains(err.Error(), "cannot use go:linkname") { + t.Fatalf("local-alias error = %v", err) } } -func TestValidateLocalityLinknameCycle(t *testing.T) { +func TestValidateLocalitiesIgnoresOrdinaryLinknameCycle(t *testing.T) { prog := NewProgram(nil) - prog.SetLinkname("example.com/p.First", "example.com/p.Second") - prog.SetLinkname("example.com/p.Second", "example.com/p.First") - prog.SetLocalityInfo("example.com/p.First", LocalityInfo{Locality: ThreadLocal}) - if err := prog.ValidateLocalities("example.com/p"); err == nil || !strings.Contains(err.Error(), "linkname cycle") { - t.Fatalf("linkname cycle error = %v", err) + prog.SetLinkname("example.com/p.first", "example.com/p.second") + prog.SetLinkname("example.com/p.second", "example.com/p.first") + if err := prog.ValidateLocalities("example.com/p"); err != nil { + t.Fatalf("ordinary linkname cycle affected locality validation: %v", err) } } -func TestValidateLocalityAllowsSelfLinkname(t *testing.T) { +func TestValidateLocalitySelfLinkname(t *testing.T) { prog := NewProgram(nil) - name := "example.com/p.Value" + name := "example.com/p.value" prog.SetLinkname(name, name) - prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal}) if err := prog.ValidateLocalities("example.com/p"); err != nil { t.Fatal(err) } - if _, got, ok, err := prog.ResolveLocality(name); err != nil || !ok || got.Locality != ThreadLocal { - t.Fatalf("ResolveLocality(%q) = %+v, %v", name, got, ok) + if canonical, _, ok, err := prog.ResolveLocality(name); err != nil || canonical != name || ok { + t.Fatalf("ordinary self-link ResolveLocality(%q) = %q, %v, %v", name, canonical, ok, err) + } + prog.SetLocalityInfo(name, LocalityInfo{Locality: ThreadLocal}) + if err := prog.ValidateLocalities("example.com/p"); err == nil || !strings.Contains(err.Error(), "cannot use go:linkname") { + t.Fatalf("local self-linkname error = %v", err) } } diff --git a/test/llgoext/locality_test.go b/test/llgoext/locality_test.go index 796f320d47..3f839419ba 100644 --- a/test/llgoext/locality_test.go +++ b/test/llgoext/locality_test.go @@ -105,7 +105,7 @@ func TestTLSAndGLSIsolation(t *testing.T) { } } -func TestLocalPackageMoveToFront(t *testing.T) { +func TestLocalPackageDirectCaches(t *testing.T) { type result struct { local int imported int @@ -113,13 +113,13 @@ func TestLocalPackageMoveToFront(t *testing.T) { done := make(chan result) go func() { glsCounter = 41 - localityscope.First = 51 + localityscope.SetFirst(51) glsCounter++ - localityscope.First++ - done <- result{local: glsCounter, imported: localityscope.First} + imported := localityscope.IncrementFirst() + done <- result{local: glsCounter, imported: imported} }() if got := <-done; got != (result{local: 42, imported: 52}) { - t.Fatalf("local package values after move-to-front = %+v", got) + t.Fatalf("local package values through direct caches = %+v", got) } } @@ -375,12 +375,12 @@ func TestInitializerScopeRunsOncePerPackageKind(t *testing.T) { } done := make(chan result) go func() { - firstValue := localityscope.First + firstValue := localityscope.First() firstCalls := localityscope.FirstCalls() secondCalls := localityscope.SecondCalls() - _ = localityscope.First + _ = localityscope.First() firstCallsAgain := localityscope.FirstCalls() - secondValue := localityscope.Second + secondValue := localityscope.Second() done <- result{ firstValue: firstValue, firstCalls: firstCalls, @@ -411,9 +411,9 @@ func TestMultiValueInitializerUsesOneGroup(t *testing.T) { } done := make(chan result) go func() { - first := localityscope.PairFirst + first := localityscope.PairFirst() afterFirst := localityscope.PairCalls() - second := localityscope.PairSecond + second := localityscope.PairSecond() done <- result{first, second, afterFirst, localityscope.PairCalls()} }() got := <-done @@ -432,12 +432,12 @@ func TestCrossPackageMixedInitializerGroup(t *testing.T) { } done := make(chan result) go func() { - scalar := localityscope.MixedScalar - address := &localityscope.MixedScalar + scalar := localityscope.MixedScalar() + address := localityscope.MixedScalarAddress() done <- result{ scalar: scalar, - pointer: localityscope.MixedPointer, - addressStable: address == &localityscope.MixedScalar, + pointer: localityscope.MixedPointer(), + addressStable: address == localityscope.MixedScalarAddress(), calls: localityscope.MixedCalls(), } }() @@ -499,6 +499,11 @@ func bumpGLSPackageBlock() int { return benchmarkGLSPackage.value } +//go:noinline +func readGLSPackageBlock() uintptr { + return uintptr(benchmarkGLSPackage.value) +} + func BenchmarkOrdinaryGlobal(b *testing.B) { value := 0 for i := 0; i < b.N; i++ { @@ -583,6 +588,19 @@ func BenchmarkComparableGLSPackageRead(b *testing.B) { benchmarkReadSink = value } +func BenchmarkAlternatingGLSPackageRead(b *testing.B) { + localitybench.PrepareReads() + benchmarkGLSPackage.pointer = &benchmarkSink + benchmarkGLSPackage.value = 1 + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += readGLSPackageBlock() + value += localitybench.ReadGLSPackage() + } + benchmarkReadSink = value +} + func BenchmarkGoroutineEntry(b *testing.B) { for i := 0; i < b.N; i++ { done := make(chan struct{}) diff --git a/test/llgoext/localitymulti/locality_test.go b/test/llgoext/localitymulti/locality_test.go new file mode 100644 index 0000000000..de1d8afe3e --- /dev/null +++ b/test/llgoext/localitymulti/locality_test.go @@ -0,0 +1,115 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package localitymulti + +import ( + "testing" + + localityblock0 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p0" + localityblock1 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p1" + localityblock2 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p2" + localityblock3 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p3" + localityblock4 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p4" + localityblock5 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p5" + localityblock6 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p6" + localityblock7 "github.com/goplus/llgo/test/llgoext/testdata/localityblocks/p7" +) + +var benchmarkSink uintptr + +func TestGLSPackageWorkingSet(t *testing.T) { + prepare := []func(){ + localityblock0.Prepare, + localityblock1.Prepare, + localityblock2.Prepare, + localityblock3.Prepare, + localityblock4.Prepare, + localityblock5.Prepare, + localityblock6.Prepare, + localityblock7.Prepare, + } + read := []func() uintptr{ + localityblock0.Read, + localityblock1.Read, + localityblock2.Read, + localityblock3.Read, + localityblock4.Read, + localityblock5.Read, + localityblock6.Read, + localityblock7.Read, + } + for i := range prepare { + prepare[i]() + if got := read[i](); got == 0 { + t.Fatalf("package %d GLS pointer is nil", i) + } + } +} + +func BenchmarkGLSPackageWorkingSet2(b *testing.B) { + localityblock0.Prepare() + localityblock1.Prepare() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localityblock0.Read() + value += localityblock1.Read() + } + benchmarkSink = value +} + +func BenchmarkGLSPackageWorkingSet4(b *testing.B) { + localityblock0.Prepare() + localityblock1.Prepare() + localityblock2.Prepare() + localityblock3.Prepare() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localityblock0.Read() + value += localityblock1.Read() + value += localityblock2.Read() + value += localityblock3.Read() + } + benchmarkSink = value +} + +func BenchmarkGLSPackageWorkingSet8(b *testing.B) { + localityblock0.Prepare() + localityblock1.Prepare() + localityblock2.Prepare() + localityblock3.Prepare() + localityblock4.Prepare() + localityblock5.Prepare() + localityblock6.Prepare() + localityblock7.Prepare() + b.ResetTimer() + var value uintptr + for i := 0; i < b.N; i++ { + value += localityblock0.Read() + value += localityblock1.Read() + value += localityblock2.Read() + value += localityblock3.Read() + value += localityblock4.Read() + value += localityblock5.Read() + value += localityblock6.Read() + value += localityblock7.Read() + } + benchmarkSink = value +} diff --git a/test/llgoext/testdata/localityblocks/p0/block.go b/test/llgoext/testdata/localityblocks/p0/block.go new file mode 100644 index 0000000000..b4978171e8 --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p0/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p0 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p1/block.go b/test/llgoext/testdata/localityblocks/p1/block.go new file mode 100644 index 0000000000..1c83a1dc19 --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p1/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p1 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p2/block.go b/test/llgoext/testdata/localityblocks/p2/block.go new file mode 100644 index 0000000000..440411923c --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p2/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p2 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p3/block.go b/test/llgoext/testdata/localityblocks/p3/block.go new file mode 100644 index 0000000000..2da455b714 --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p3/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p3 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p4/block.go b/test/llgoext/testdata/localityblocks/p4/block.go new file mode 100644 index 0000000000..82fa5ee80d --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p4/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p4 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p5/block.go b/test/llgoext/testdata/localityblocks/p5/block.go new file mode 100644 index 0000000000..8e5151e02d --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p5/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p5 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p6/block.go b/test/llgoext/testdata/localityblocks/p6/block.go new file mode 100644 index 0000000000..e11b82c5d2 --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p6/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p6 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityblocks/p7/block.go b/test/llgoext/testdata/localityblocks/p7/block.go new file mode 100644 index 0000000000..6ebff03a61 --- /dev/null +++ b/test/llgoext/testdata/localityblocks/p7/block.go @@ -0,0 +1,31 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package p7 + +import "unsafe" + +var backing int + +//llgo:gls +var pointer *int + +func Prepare() { pointer = &backing } + +//go:noinline +func Read() uintptr { return uintptr(unsafe.Pointer(pointer)) } diff --git a/test/llgoext/testdata/localityscope/scope.go b/test/llgoext/testdata/localityscope/scope.go index dce9828dca..e20bd0a86a 100644 --- a/test/llgoext/testdata/localityscope/scope.go +++ b/test/llgoext/testdata/localityscope/scope.go @@ -32,13 +32,22 @@ func initSecond() int { } //llgo:gls -var First = initFirst() +var first = initFirst() //llgo:gls -var Second = initSecond() +var second = initSecond() func FirstCalls() int { return firstCalls } func SecondCalls() int { return secondCalls } +func First() int { return first } +func Second() int { return second } + +func SetFirst(value int) { first = value } + +func IncrementFirst() int { + first++ + return first +} var pairCalls int @@ -48,10 +57,13 @@ func initPair() (int, int) { } //llgo:gls -var PairFirst, PairSecond = initPair() +var pairFirst, pairSecond = initPair() func PairCalls() int { return pairCalls } +func PairFirst() int { return pairFirst } +func PairSecond() int { return pairSecond } + var mixedCalls int var mixedBacking = 500 @@ -61,6 +73,11 @@ func initMixed() (int, *int) { } //llgo:tls -var MixedScalar, MixedPointer = initMixed() +var mixedScalar, mixedPointer = initMixed() func MixedCalls() int { return mixedCalls } + +func MixedScalar() int { return mixedScalar } +func MixedPointer() *int { return mixedPointer } + +func MixedScalarAddress() *int { return &mixedScalar } From 75cd0b5fa6dd9a16e33495ee152a9d2e188e0150 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 16 Jul 2026 02:51:12 +0800 Subject: [PATCH 12/13] cl: cover locality validation and import lowering --- cl/locality.go | 16 +++------------- cl/locality_lower_test.go | 31 +++++++++++++++++++++++++++++++ cl/locality_test.go | 27 +++++++++++++++++++++++++++ ssa/locality_test.go | 9 +++++++-- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/cl/locality.go b/cl/locality.go index 0961685f30..d9c561ae39 100644 --- a/cl/locality.go +++ b/cl/locality.go @@ -43,19 +43,16 @@ func PrepareLocalVariables(prog llssa.Program, fset *token.FileSet, pkg *types.P for name, local := range prepared { prog.SetLocalityInfo(llssa.FullName(pkg, name), local) } - for fullName, local := range prog.PackageLocalities(path) { + for fullName := range prog.PackageLocalities(path) { name := strings.TrimPrefix(fullName, path+".") object, _ := pkg.Scope().Lookup(name).(*types.Var) if object == nil { return fmt.Errorf("locality layout: package %s has no variable %s", path, name) } - canonical, _, _, err := prog.ResolveLocality(fullName) + _, _, _, err := prog.ResolveLocality(fullName) if err != nil { return err } - if canonical != fullName && local.HasInitializer { - return fmt.Errorf("locality layout: linkname alias %s cannot have an initializer", fullName) - } prog.SetLocalStorage(fullName, localitylayout.StorageForType(object.Type())) } _, err = planLocalPackage(prog, pkg) @@ -84,17 +81,10 @@ func planLocalPackage(prog llssa.Program, pkg *types.Package) (localitylayout.Pa decls := prog.PackageLocalities(path) input := make([]localitylayout.Declaration, 0, len(decls)) for fullName := range decls { - canonical, info, _, err := prog.ResolveLocality(fullName) + _, info, _, err := prog.ResolveLocality(fullName) if err != nil { return localitylayout.Package{}, err } - if canonical != fullName { - target, targetOK := prog.VariableLocality(canonical) - if !targetOK || target.Locality == locality.None { - return localitylayout.Package{}, fmt.Errorf("locality layout: linkname target %s for %s is not a local variable", canonical, fullName) - } - continue - } name := strings.TrimPrefix(fullName, prefix) object, _ := pkg.Scope().Lookup(name).(*types.Var) if object == nil { diff --git a/cl/locality_lower_test.go b/cl/locality_lower_test.go index 8d16ae9e26..8bae3152f3 100644 --- a/cl/locality_lower_test.go +++ b/cl/locality_lower_test.go @@ -73,6 +73,9 @@ func TestLocalityLoweringResolution(t *testing.T) { if got := ctx.localTypesPackage("example.com/loaded.Value"); got != loaded { t.Fatalf("loaded localTypesPackage = %v, want %v", got, loaded) } + if got := (&context{goProg: global.Pkg.Prog}).localTypesPackage(name); got != typesPkg { + t.Fatalf("SSA-program localTypesPackage = %v, want %v", got, typesPkg) + } if got := (&context{}).localTypesPackage("example.com/missing.Value"); got != nil { t.Fatalf("missing localTypesPackage = %v", got) } @@ -85,6 +88,34 @@ func TestLocalityLoweringResolution(t *testing.T) { } } +func TestLocalityLoweringDeclarationOnlyState(t *testing.T) { + typesPkg, global := localitySSAGlobal(t, "example.com/declaration") + name := llssa.FullName(typesPkg, global.Name()) + prog := ssatest.NewProgram(t, nil) + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + prog.SetLocalStorage(name, llssa.LocalStorageNativeTLS) + llvmPkg := prog.NewPackage(typesPkg.Name(), typesPkg.Path()) + ctx := &context{ + prog: prog, + pkg: llvmPkg, + goTyps: typesPkg, + locality: localityLowering{ + variables: make(map[*ssa.Global]*localVariable), + }, + } + + addr := ctx.localVariableAddr(nil, global, llssa.VariableLocality{Info: llssa.LocalityInfo{Locality: llssa.ThreadLocal}}, name) + if addr != ctx.locality.variables[global].owner.direct[name].Expr { + t.Fatal("localVariableAddr did not return declaration-only TLS storage") + } + + owner := &localPackage{plan: localitylayout.Package{Path: typesPkg.Path()}} + initializer := ctx.buildLocalInitializer(llvmPkg, owner, locality.Thread, []localitylayout.Initializer{{Name: typesPkg.Path() + ".initLocal", Order: 1}}, false) + if initializer.dispatch.HasBody() || initializer.ensure.HasBody() { + t.Fatal("declaration-only initializer unexpectedly defined a body") + } +} + func TestLocalityLoweringDiagnostics(t *testing.T) { typesPkg, global := localitySSAGlobal(t, "example.com/diagnostic") name := llssa.FullName(typesPkg, global.Name()) diff --git a/cl/locality_test.go b/cl/locality_test.go index 6c0c6ea515..877313838c 100644 --- a/cl/locality_test.go +++ b/cl/locality_test.go @@ -563,6 +563,33 @@ var alias int } } +func TestNewPackageValidatesPreloadedLocalityMetadata(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "locality.go", `package locality +var value int +`, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + info := newLocalityTypeInfo() + pkg, err := (&types.Config{}).Check("example.com/locality", fset, files, info) + if err != nil { + t.Fatal(err) + } + goProg := ssa.NewProgram(fset, ssa.SanityCheckFunctions) + ssaPkg := goProg.CreatePackage(pkg, files, info, true) + ssaPkg.Build() + + prog := ssatest.NewProgram(t, nil) + name := llssa.FullName(pkg, "value") + prog.SetLocalityInfo(name, llssa.LocalityInfo{Locality: llssa.ThreadLocal}) + prog.SetLinkname(name, name+"Alias") + if _, err := NewPackage(prog, ssaPkg, files); err == nil || !strings.Contains(err.Error(), "cannot use go:linkname") { + t.Fatalf("NewPackage locality validation error = %v", err) + } +} + func TestParseRejectsExportedLocalVariable(t *testing.T) { fset := token.NewFileSet() file, err := parser.ParseFile(fset, "locality.go", `package locality diff --git a/ssa/locality_test.go b/ssa/locality_test.go index e931797443..b15f59e513 100644 --- a/ssa/locality_test.go +++ b/ssa/locality_test.go @@ -111,11 +111,16 @@ func TestRejectsLinknameLocality(t *testing.T) { func TestValidateLocalitiesIgnoresOrdinaryLinknameCycle(t *testing.T) { prog := NewProgram(nil) - prog.SetLinkname("example.com/p.first", "example.com/p.second") - prog.SetLinkname("example.com/p.second", "example.com/p.first") + first := "example.com/p.first" + second := "example.com/p.second" + prog.SetLinkname(first, second) + prog.SetLinkname(second, first) if err := prog.ValidateLocalities("example.com/p"); err != nil { t.Fatalf("ordinary linkname cycle affected locality validation: %v", err) } + if _, _, _, err := prog.ResolveLocality(first); err == nil || !strings.Contains(err.Error(), "linkname cycle") { + t.Fatalf("ResolveLocality cycle error = %v", err) + } } func TestValidateLocalitySelfLinkname(t *testing.T) { From 4f46588a2cfb4c102c57f2e5826d3b7bf43897d4 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 23 Jul 2026 18:23:38 +0800 Subject: [PATCH 13/13] compiler/runtime: address locality review feedback --- cl/locality_lower.go | 2 ++ runtime/internal/runtime/local_context.go | 6 ++++++ runtime/internal/runtime/local_initializer.go | 2 ++ ssa/locality.go | 20 ++++++++++++++----- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cl/locality_lower.go b/cl/locality_lower.go index b06dc2e432..7fbf599e64 100644 --- a/cl/locality_lower.go +++ b/cl/locality_lower.go @@ -28,6 +28,8 @@ import ( "golang.org/x/tools/go/ssa" ) +// localInitReady is part of the compiler/runtime ABI. Keep it in sync with +// runtime/internal/runtime.localInitReady. const localInitReady = 2 type localVariable struct { diff --git a/runtime/internal/runtime/local_context.go b/runtime/internal/runtime/local_context.go index 5425984bb6..8239163a85 100644 --- a/runtime/internal/runtime/local_context.go +++ b/runtime/internal/runtime/local_context.go @@ -35,6 +35,9 @@ type localBlock struct { cacheSlot *uintptr } +// EnterLocalContext installs ctx when the current thread has no local owner. +// A nonzero result means this is a nested Go entry that inherited the returned +// context; in that case ctx is not installed. func EnterLocalContext(ctx *LocalContext) uintptr { previous := currentLocalContext if previous == 0 { @@ -46,6 +49,9 @@ func EnterLocalContext(ctx *LocalContext) uintptr { return previous } +// LeaveLocalContext finishes an entry paired with EnterLocalContext. A nested +// entry verifies and retains its inherited context. An outer entry clears ctx +// and releases its package-block roots. func LeaveLocalContext(ctx *LocalContext, previous uintptr) { if previous != 0 { if currentLocalContext != previous { diff --git a/runtime/internal/runtime/local_initializer.go b/runtime/internal/runtime/local_initializer.go index 71fa122eda..a0aeac2ebb 100644 --- a/runtime/internal/runtime/local_initializer.go +++ b/runtime/internal/runtime/local_initializer.go @@ -18,6 +18,8 @@ package runtime import "unsafe" +// The compiler emits localInitReady directly from cl/locality_lower.go. Keep +// these numeric values stable and update the compiler constant if they change. const ( localInitUninitialized uint8 = iota localInitInitializing diff --git a/ssa/locality.go b/ssa/locality.go index a8e9a62a7c..897d5b94bd 100644 --- a/ssa/locality.go +++ b/ssa/locality.go @@ -103,18 +103,21 @@ func resolveLocality(lookup func(string) (VariableLocality, bool), linkname func if !ok { result = VariableLocality{} } - seen := make(map[string]bool) + var seen map[string]bool current := name for { - if seen[current] { - return "", VariableLocality{}, false, fmt.Errorf("declaration linkname cycle involving %s", current) - } - seen[current] = true target, hasLink := linkname(current) target = strings.TrimPrefix(target, "go:") if !hasLink || target == "" { return current, result, ok, nil } + if seen == nil { + seen = make(map[string]bool) + } + if seen[current] { + return "", VariableLocality{}, false, fmt.Errorf("declaration linkname cycle involving %s", current) + } + seen[current] = true if currentInfo, exists := lookup(current); exists && currentInfo.Locality != locality.None { return "", VariableLocality{}, false, fmt.Errorf("local variable %s cannot use go:linkname", current) } @@ -135,6 +138,10 @@ func hasInitialization(info locality.Info) bool { func (p Program) ValidateLocalities(pkgPath string) error { prefix := pkgPath + "." p.localities.mu.RLock() + if len(p.localities.entries) == 0 { + p.localities.mu.RUnlock() + return nil + } nameSet := make(map[string]bool) localNames := make(map[string]bool) for name, info := range p.localities.entries { @@ -146,6 +153,9 @@ func (p Program) ValidateLocalities(pkgPath string) error { } } p.localities.mu.RUnlock() + if len(localNames) == 0 { + return nil + } p.linknameMu.RLock() links := make(map[string]string, len(p.linkname)) for name, target := range p.linkname {