diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0eb5994..1c9e598 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -58,7 +58,7 @@ jobs:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
ARTIFACT: ${{ matrix.artifact }}
- LATTICE_COMPAT_SERVER_MIN: v0.2.2-alpha.2
+ LATTICE_COMPAT_SERVER_MIN: v0.2.2-alpha.19
LATTICE_COMPAT_DASHBOARD_MIN: v0.2.2-alpha.7
run: |
VERSION="${GITHUB_REF_NAME#v}"
diff --git a/cmd/lattice-agent/guard_reality_test.go b/cmd/lattice-agent/guard_reality_test.go
index 8e57d6e..0bdfa18 100644
--- a/cmd/lattice-agent/guard_reality_test.go
+++ b/cmd/lattice-agent/guard_reality_test.go
@@ -65,7 +65,7 @@ func TestWriteGuardManagedSHAOnlyOutputsCanonicalHashOnSuccess(t *testing.T) {
func TestReportedCapabilitiesAdvertiseGuardManagedSHA(t *testing.T) {
got := reportedCapabilities()
- if !reflect.DeepEqual(got, []string{guardManagedSHACapability}) {
+ if !reflect.DeepEqual(got, []string{durableTaskResultCapability, guardManagedSHACapability}) {
t.Fatalf("reported capabilities = %#v", got)
}
}
diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go
new file mode 100644
index 0000000..407ff01
--- /dev/null
+++ b/cmd/lattice-agent/linechain_e2e_test.go
@@ -0,0 +1,1243 @@
+//go:build linechain_e2e && (darwin || linux || freebsd || openbsd || netbsd)
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/LatticeNet/lattice-node-agent/internal/linechain"
+ "github.com/LatticeNet/lattice-node-agent/internal/singboxdiscover"
+ "github.com/LatticeNet/lattice-node-agent/internal/taskoutbox"
+ "github.com/LatticeNet/lattice-sdk/model"
+)
+
+const (
+ linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN"
+ linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT"
+ linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR"
+ linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR"
+ linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT"
+ linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK"
+ linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE"
+ linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER"
+ linechainE2ERecoveryResult = "LATTICE_LINECHAIN_E2E_RECOVERY_RESULT"
+ linechainE2EResolveResult = "LATTICE_LINECHAIN_E2E_RESOLVE_RESULT"
+ linechainE2EInventoryResult = "LATTICE_LINECHAIN_E2E_INVENTORY_RESULT"
+ linechainE2EOutboxEnv = "LATTICE_LINECHAIN_E2E_OUTBOX"
+ linechainE2ETaskJSONEnv = "LATTICE_LINECHAIN_E2E_TASK_JSON"
+ linechainE2EBeginResultEnv = "LATTICE_LINECHAIN_E2E_BEGIN_RESULT"
+ linechainE2EAckResultEnv = "LATTICE_LINECHAIN_E2E_ACK_RESULT"
+)
+
+// TestLinechainE2EBeginHelper durably records the exact server lease before
+// its script is allowed to execute.
+func TestLinechainE2EBeginHelper(t *testing.T) {
+ resultPath := os.Getenv(linechainE2EBeginResultEnv)
+ if resultPath == "" {
+ return
+ }
+ var task model.Task
+ raw, err := os.ReadFile(mustAbsoluteEnv(t, linechainE2ETaskJSONEnv))
+ if err != nil || json.Unmarshal(raw, &task) != nil {
+ t.Fatalf("decode exact leased task: %v", err)
+ }
+ outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer outbox.Close()
+ committed, err := outbox.BeginWithProtocol(task, "linechain-e3-v2")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if committed {
+ if err := outbox.ConfirmDurability(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ writeE2EJSON(t, resultPath, map[string]any{"committed": committed, "task_id": task.ID, "lease_id": task.LeaseID})
+}
+
+// TestLinechainE2EAckHelper durably removes only the exact result acknowledged
+// by the server after its successful replay response.
+func TestLinechainE2EAckHelper(t *testing.T) {
+ resultPath := os.Getenv(linechainE2EAckResultEnv)
+ if resultPath == "" {
+ return
+ }
+ taskID, leaseID := mustEnv(t, linechainE2ETaskEnv), mustEnv(t, linechainE2ELeaseEnv)
+ outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer outbox.Close()
+ entry := pendingE2EEntry(t, outbox, taskID, leaseID)
+ if err := outbox.Remove(entry); err != nil {
+ t.Fatal(err)
+ }
+ writeE2EJSON(t, resultPath, map[string]string{"task_id": taskID, "lease_id": leaseID})
+}
+
+var managedE2EProcesses = struct {
+ sync.Mutex
+ done map[int]<-chan error
+}{done: make(map[int]<-chan error)}
+
+// TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. The
+// script makes the real 1.13.x binary mandatory, so this test never skips.
+func TestLinechainRealSingBoxE2E(t *testing.T) {
+ bin := requireSingBoxE2EBinary(t)
+ root := os.Getenv(linechainE2ERootEnv)
+ if !filepath.IsAbs(root) {
+ t.Fatalf("%s must be an absolute test root", linechainE2ERootEnv)
+ }
+ if err := os.MkdirAll(root, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ origin := startEchoOrigin(t)
+ aPort, bPort, clientPort := freePort(t), freePort(t), freePort(t)
+ observer := startTCPObserver(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(aPort)))
+ decoy := httptest.NewTLSServer(nil)
+ t.Cleanup(decoy.Close)
+ decoyAddress := strings.TrimPrefix(decoy.URL, "https://")
+ decoyHost, decoyPortText, err := net.SplitHostPort(decoyAddress)
+ if err != nil {
+ t.Fatal(err)
+ }
+ decoyPort, err := strconv.Atoi(decoyPortText)
+ if err != nil {
+ t.Fatal(err)
+ }
+ realityPrivate, realityPublic := generateRealityKeypair(t, bin)
+ const (
+ uuidA = "11111111-1111-4111-8111-111111111111"
+ uuidB = "22222222-2222-4222-8222-222222222222"
+ lineUUIDA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
+ lineUUIDB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
+ basename = "lattice-linechain-0123456789abcdef0123.json"
+ )
+
+ aDir := filepath.Join(root, "a")
+ bDir := filepath.Join(root, "b")
+ clientDir := filepath.Join(root, "client")
+ mustMkdir(t, aDir)
+ mustMkdir(t, bDir)
+ mustMkdir(t, clientDir)
+ writeFile(t, filepath.Join(aDir, "config.json"), fmt.Sprintf(`{
+ "log":{"level":"error"},
+ "inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q,"flow":"xtls-rprx-vision"}],"tls":{"enabled":true,"server_name":"e2e.lattice.invalid","reality":{"enabled":true,"handshake":{"server":%q,"server_port":%d},"private_key":%q,"short_id":["0123456789abcdef"]}}}],
+ "outbounds":[{"type":"direct","tag":"direct"}],
+ "route":{"rules":[{"inbound":["target-a"],"outbound":"direct"}]}
+}
+`, aPort, uuidA, decoyHost, decoyPort, realityPrivate))
+ writeFile(t, filepath.Join(bDir, "config.json"), fmt.Sprintf(`{
+ "log":{"level":"error"},
+ "inbounds":[{"type":"vless","tag":"source-b","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q}]}],
+ "outbounds":[{"type":"direct","tag":"direct"}],
+ "route":{"final":"direct"}
+}
+`, bPort, uuidB))
+ writeFile(t, filepath.Join(clientDir, "config.json"), fmt.Sprintf(`{
+ "log":{"level":"error"},
+ "inbounds":[{"type":"socks","tag":"client","listen":"127.0.0.1","listen_port":%d}],
+ "outbounds":[{"type":"vless","tag":"to-b","server":"127.0.0.1","server_port":%d,"uuid":%q}],
+ "route":{"rules":[{"inbound":["client"],"outbound":"to-b"}]}
+}
+`, clientPort, bPort, uuidB))
+
+ startManagedSingBox(t, bin, root, "a", aDir, aPort)
+ if err := verifyManagedSingBox(root, "a", aPort); err != nil {
+ t.Fatalf("managed target A is inactive: %v", err)
+ }
+ startManagedSingBox(t, bin, root, "b", bDir, bPort)
+ startManagedSingBox(t, bin, root, "client", clientDir, clientPort)
+
+ assertSOCKSEcho(t, clientPort, origin)
+ if got := observer.Count(); got != 0 {
+ t.Fatalf("B began chained: observer accepted %d connections before apply", got)
+ }
+
+ sidecarPath := filepath.Join(root, "lattice-metadata.json")
+ fragmentPath := filepath.Join(bDir, basename)
+ txnDir := filepath.Join(root, "txn")
+ fragment := fmt.Sprintf(`{
+ "outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":%q,"flow":"xtls-rprx-vision","tls":{"enabled":true,"server_name":"e2e.lattice.invalid","utls":{"enabled":true,"fingerprint":"chrome"},"reality":{"enabled":true,"public_key":%q,"short_id":"0123456789abcdef"}}}],
+ "route":{"rules":[{"inbound":["source-b"],"outbound":"chain-to-a"}]}
+}
+`, observer.Port(), uuidA, realityPublic)
+ initialSidecar := canonicalJSONString(t, fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","unknown_root":{"keep":true},"inbounds":[{"tag":"unrelated-before","line_uuid":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","keep":1},{"tag":"source-b","line_uuid":%q,"ordinary":"keep"},{"tag":"unrelated-after","line_uuid":"dddddddd-dddd-4ddd-8ddd-dddddddddddd","keep":2}]}`, lineUUIDB))
+ desiredSidecar := canonicalJSONString(t, fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","unknown_root":{"keep":true},"inbounds":[{"tag":"unrelated-before","line_uuid":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","keep":1},{"tag":"source-b","line_uuid":%q,"ordinary":"keep","chain":{"downstream_line_uuid":%q}},{"tag":"unrelated-after","line_uuid":"dddddddd-dddd-4ddd-8ddd-dddddddddddd","keep":2}]}`, lineUUIDB, lineUUIDA))
+ writeFile(t, sidecarPath, initialSidecar)
+ targetLineUUID := lineUUIDA
+
+ m := openE2EManager(t, bin, root, bDir, sidecarPath, txnDir, bPort)
+ defer m.Close()
+
+ // Crash a real apply helper process group after it publishes both artifacts,
+ // while this test (the agent/supervisor analogue) remains alive. Recovery
+ // must restore and restart B before any inventory, traffic, or result callback.
+ crashDoc := bindE2EDocument("create", basename, "", &fragment, lineUUIDB, "source-b", nil, &targetLineUUID)
+ crashBytes := marshalE2EDocument(t, crashDoc)
+ apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--", root)
+ crashMarker := filepath.Join(root, "restart-blocked")
+ apply.Env = append(os.Environ(),
+ linechainE2EBinEnv+"="+bin,
+ linechainE2ERootEnv+"="+root,
+ linechainE2EConfigEnv+"="+bDir,
+ linechainE2ESidecarEnv+"="+sidecarPath,
+ linechainE2EBPortEnv+"="+strconv.Itoa(bPort),
+ linechainE2ETaskEnv+"=crash-task",
+ linechainE2ELeaseEnv+"=crash-lease",
+ "LATTICE_LINECHAIN_TASK_SCRIPT_SHA256="+digestText("e2e-helper-script"),
+ linechainE2ECrashMarkerEnv+"="+crashMarker,
+ )
+ apply.Stdin = bytes.NewReader(crashBytes)
+ apply.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ var applyLog bytes.Buffer
+ apply.Stdout, apply.Stderr = &applyLog, &applyLog
+ if err := apply.Start(); err != nil {
+ t.Fatal(err)
+ }
+ applyDone := make(chan error, 1)
+ go func() { applyDone <- apply.Wait() }()
+ var cleanupApply sync.Once
+ stopApply := func() {
+ cleanupApply.Do(func() {
+ _ = syscall.Kill(-apply.Process.Pid, syscall.SIGKILL)
+ select {
+ case <-applyDone:
+ case <-time.After(5 * time.Second):
+ t.Errorf("apply helper did not exit after process-group kill")
+ }
+ killMarkerProcess(t, crashMarker)
+ })
+ }
+ t.Cleanup(stopApply)
+ waitForFileOrProcess(t, crashMarker, applyDone, &applyLog)
+ waitForExactPair(t, fragmentPath, fragment, sidecarPath, desiredSidecar)
+ stopApply()
+
+ order := []string{}
+ if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error {
+ order = append(order, "result")
+ if result.ExitCode == 0 || !strings.Contains(result.Error, "interrupted") {
+ return fmt.Errorf("unexpected recovered result: %+v", result)
+ }
+ return nil
+ }, "node-b"); err != nil {
+ t.Fatalf("recover killed helper: %v (helper output %s)", err, applyLog.String())
+ }
+ order = append(order, "inventory")
+ assertUnchainedInventory(t, bDir, sidecarPath)
+ order = append(order, "traffic")
+ assertSOCKSEcho(t, clientPort, origin)
+ if strings.Join(order, ",") != "result,inventory,traffic" {
+ t.Fatalf("recovery ordering = %v", order)
+ }
+ if got := observer.Count(); got != 0 {
+ t.Fatalf("recovery exposed chained traffic before successful apply: %d", got)
+ }
+
+ applyAndResolve(t, m, marshalE2EDocument(t, crashDoc), "create-task", "create-lease")
+ if err := verifyManagedSingBox(root, "a", aPort); err != nil {
+ t.Fatalf("managed target A became inactive after apply: %v", err)
+ }
+ assertSOCKSEcho(t, clientPort, origin)
+ if got := observer.Count(); got == 0 {
+ t.Fatal("traffic did not traverse the B-to-A observer after apply")
+ }
+ assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port())
+
+ // Simulate the ordinary independent metadata writer changing unrelated
+ // sidecar bytes. The declared edge remains intact, and remove must tolerate
+ // this non-E3 sidecar drift instead of applying a stale sidecar CAS.
+ resyncedSidecar := fmt.Sprintf(`{"generated_by":"ordinary-resync","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}],"schema":"lattice.singbox-metadata.v2"}
+`, lineUUIDB, lineUUIDA)
+ writeFile(t, sidecarPath, resyncedSidecar)
+ assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port())
+
+ removeDoc := bindE2EDocument("remove", basename, digestText(fragment), nil, lineUUIDB, "source-b", &targetLineUUID, nil)
+ applyAndResolve(t, m, marshalE2EDocument(t, removeDoc), "remove-task", "remove-lease")
+ if _, err := os.Stat(fragmentPath); !os.IsNotExist(err) {
+ t.Fatalf("removed fragment still exists: %v", err)
+ }
+ assertUnchainedInventory(t, bDir, sidecarPath)
+ before := observer.Count()
+ assertSOCKSEcho(t, clientPort, origin)
+ if got := observer.Count(); got != before {
+ t.Fatalf("traffic still used A after remove: observer %d -> %d", before, got)
+ }
+}
+
+// TestLinechainE2EApplyHelper is a child-only entry point used to create a real
+// crash boundary. The parent test kills this process group, not the supervisor.
+func TestLinechainE2EApplyHelper(t *testing.T) {
+ if os.Getenv(linechainE2ETaskEnv) == "" {
+ return
+ }
+ m, err := linechain.OpenHelper(filepath.Join(os.Getenv(linechainE2ERootEnv), "txn"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ configureE2EManager(t, m)
+ if err := m.Apply(context.Background(), os.Stdin, os.Getenv(linechainE2ETaskEnv), os.Getenv(linechainE2ELeaseEnv), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestLinechainE2ERecoverHelper is a child-only startup recovery entry point.
+// It writes the recovered result only after RequireRecovered has restored the
+// old artifacts and restarted the managed B process.
+func TestLinechainE2ERecoverHelper(t *testing.T) {
+ resultPath := os.Getenv(linechainE2ERecoveryResult)
+ if resultPath == "" {
+ return
+ }
+ root := mustAbsoluteEnv(t, linechainE2ERootEnv)
+ taskID, leaseID := mustEnv(t, linechainE2ETaskEnv), mustEnv(t, linechainE2ELeaseEnv)
+ consumeE2ECrashMarker(t, mustAbsoluteEnv(t, linechainE2ECrashMarkerEnv))
+ m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv))
+ defer m.Close()
+ outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer outbox.Close()
+ authority, err := captureLinechainAuthority(m, outbox)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.RequireRecoveredAuthorized(context.Background(), func(result model.TaskResult) error {
+ committed, completeErr := outbox.Complete(result)
+ if completeErr != nil && !committed {
+ return completeErr
+ }
+ return outbox.ConfirmDurability()
+ }, "node-b", authority); err != nil {
+ t.Fatal(err)
+ }
+ writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox, taskID, leaseID))
+}
+
+// TestLinechainE2EResolveHelper converts a successful helper run into the
+// stable durable result, cleans its journal, and only then publishes JSON.
+func TestLinechainE2EResolveHelper(t *testing.T) {
+ resultPath := os.Getenv(linechainE2EResolveResult)
+ if resultPath == "" {
+ return
+ }
+ root := mustAbsoluteEnv(t, linechainE2ERootEnv)
+ taskID := mustEnv(t, linechainE2ETaskEnv)
+ leaseID := mustEnv(t, linechainE2ELeaseEnv)
+ m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv))
+ defer m.Close()
+ outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer outbox.Close()
+ result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0, FinishedAt: time.Now().UTC()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ committed, err := outbox.Complete(result)
+ if err != nil && !committed {
+ t.Fatal(err)
+ }
+ if err := outbox.ConfirmDurability(); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Cleanup(taskID, leaseID); err != nil {
+ t.Fatal(err)
+ }
+ writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox, taskID, leaseID))
+}
+
+func pendingE2EEntry(t *testing.T, outbox *taskoutbox.Store, taskID, leaseID string) taskoutbox.Entry {
+ t.Helper()
+ entries, err := outbox.Pending()
+ if err != nil {
+ t.Fatalf("durable result handoff: entries=%+v err=%v", entries, err)
+ }
+ for _, entry := range entries {
+ if entry.Task.ID == taskID && entry.Task.LeaseID == leaseID && entry.Result != nil {
+ return entry
+ }
+ }
+ t.Fatalf("durable result %s/%s not found: %+v", taskID, leaseID, entries)
+ return taskoutbox.Entry{}
+}
+
+func pendingE2EResult(t *testing.T, outbox *taskoutbox.Store, taskID, leaseID string) model.TaskResult {
+ t.Helper()
+ return *pendingE2EEntry(t, outbox, taskID, leaseID).Result
+}
+
+func TestLinechainE2EDurableResultSurvivesOutboxReopen(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "outbox")
+ task := model.Task{ID: "task-reopen", LeaseID: "lease-reopen", Script: "# lattice-linechain-e3-v2\nexact"}
+ outbox, err := taskoutbox.Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if committed, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil || !committed {
+ t.Fatalf("begin: committed=%v err=%v", committed, err)
+ }
+ result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}
+ if committed, err := outbox.Complete(result); err != nil || !committed {
+ t.Fatalf("complete: committed=%v err=%v", committed, err)
+ }
+ if err := outbox.ConfirmDurability(); err != nil {
+ t.Fatal(err)
+ }
+ if err := outbox.Close(); err != nil {
+ t.Fatal(err)
+ }
+ reopened, err := taskoutbox.Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reopened.Close()
+ second := model.Task{ID: "task-second", LeaseID: "lease-second", Script: "# lattice-linechain-e3-v2\nsecond"}
+ if _, err := reopened.BeginWithProtocol(second, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := reopened.Complete(model.TaskResult{TaskID: second.ID, LeaseID: second.LeaseID, ExitCode: 0}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reopened.ConfirmDurability(); err != nil {
+ t.Fatal(err)
+ }
+ if got := pendingE2EResult(t, reopened, task.ID, task.LeaseID); !sameTaskResult(got, result) {
+ t.Fatalf("reopened result changed: got=%+v want=%+v", got, result)
+ }
+ if err := reopened.Remove(pendingE2EEntry(t, reopened, task.ID, task.LeaseID)); err != nil {
+ t.Fatal(err)
+ }
+ if got := pendingE2EResult(t, reopened, second.ID, second.LeaseID); got.TaskID != second.ID {
+ t.Fatalf("ack removed wrong durable result: %+v", got)
+ }
+}
+
+func TestLinechainE2ERecoveryDoesNotCleanupBeforeDurableResult(t *testing.T) {
+ root := t.TempDir()
+ configDir, txnDir := filepath.Join(root, "conf"), filepath.Join(root, "txn")
+ if err := os.Mkdir(configDir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ sidecar := filepath.Join(root, "sidecar.json")
+ writeFile(t, sidecar, `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source-b","line_uuid":"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"}]}`)
+ m, err := linechain.Open(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ if err := m.ConfigureLayout(configDir, sidecar); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("/usr/bin/true", []string{"/usr/bin/true"}, []string{"/usr/bin/true"}); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{"outbounds":[],"route":{"rules":[]}}`
+ target := "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
+ document := marshalE2EDocument(t, bindE2EDocument("create", "lattice-linechain-0123456789abcdef0123.json", "", &fragment, "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "source-b", nil, &target))
+ script := "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '" + base64.StdEncoding.EncodeToString(document) + "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n"
+ task := model.Task{ID: "task-fault", LeaseID: "lease-fault", Script: script}
+ if err := m.Apply(context.Background(), bytes.NewReader(document), task.ID, task.LeaseID, linechainTaskScriptSHA(script)); err != nil {
+ t.Fatal(err)
+ }
+ result, err := m.ResolveAfterRun(context.Background(), task, model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0})
+ if err != nil {
+ t.Fatal(err)
+ }
+ outboxDir := filepath.Join(root, "outbox")
+ outbox, err := taskoutbox.Open(outboxDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := outbox.Complete(result); err == nil {
+ t.Fatal("Complete without Begin unexpectedly succeeded")
+ }
+ if refs, err := m.Snapshot(); err != nil || len(refs) != 1 || !refs[0].Terminal {
+ t.Fatalf("transaction cleaned before durable result: refs=%+v err=%v", refs, err)
+ }
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ authority, err := captureLinechainAuthority(m, outbox)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.RequireRecoveredAuthorized(context.Background(), func(got model.TaskResult) error {
+ if !sameTaskResult(got, result) {
+ return fmt.Errorf("recovered result changed")
+ }
+ _, err := outbox.Complete(got)
+ if err != nil {
+ return err
+ }
+ return outbox.ConfirmDurability()
+ }, "node-b", authority); err != nil {
+ t.Fatal(err)
+ }
+ if refs, err := m.Snapshot(); err != nil || len(refs) != 0 {
+ t.Fatalf("transaction survived durable recovery: refs=%+v err=%v", refs, err)
+ }
+ if got := pendingE2EResult(t, outbox, task.ID, task.LeaseID); !sameTaskResult(got, result) {
+ t.Fatalf("durable recovered result changed: %+v", got)
+ }
+}
+
+// TestLinechainE2EInventoryHelper publishes the inventory discovered from the
+// actual post-restart config directory and semantic sidecar.
+func TestLinechainE2EInventoryHelper(t *testing.T) {
+ resultPath := os.Getenv(linechainE2EInventoryResult)
+ if resultPath == "" {
+ return
+ }
+ configDir := mustAbsoluteEnv(t, linechainE2EConfigEnv)
+ paths, err := filepath.Glob(filepath.Join(configDir, "*.json"))
+ if err != nil || len(paths) == 0 {
+ t.Fatalf("discover config files: paths=%v err=%v", paths, err)
+ }
+ inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", paths, mustAbsoluteEnv(t, linechainE2ESidecarEnv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeE2EJSON(t, resultPath, inv)
+}
+
+// TestLinechainE2ERestartHelper is the fixed restart/active command used by the
+// Manager. It stops the prior B instance and starts a checked replacement.
+func TestLinechainE2ERestartHelper(t *testing.T) {
+ if os.Getenv(linechainE2ERootEnv) == "" {
+ return
+ }
+ root := mustAbsoluteEnv(t, linechainE2ERootEnv)
+ if marker := os.Getenv(linechainE2ECrashMarkerEnv); marker != "" {
+ if !filepath.IsAbs(marker) {
+ t.Fatalf("%s must be absolute", linechainE2ECrashMarkerEnv)
+ }
+ if err := os.WriteFile(marker, []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ for {
+ time.Sleep(time.Hour)
+ }
+ }
+ if err := restartManagedSingBox(mustExecutableEnv(t, linechainE2EBinEnv), root, "b", mustAbsoluteEnv(t, linechainE2EConfigEnv), mustEnvPort(linechainE2EBPortEnv)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func mustEnv(t *testing.T, key string) string {
+ t.Helper()
+ value := strings.TrimSpace(os.Getenv(key))
+ if value == "" {
+ t.Fatalf("%s is required", key)
+ }
+ return value
+}
+
+func mustAbsoluteEnv(t *testing.T, key string) string {
+ t.Helper()
+ value := mustEnv(t, key)
+ if !filepath.IsAbs(value) {
+ t.Fatalf("%s must be absolute", key)
+ }
+ return value
+}
+
+func mustExecutableEnv(t *testing.T, key string) string {
+ t.Helper()
+ value := mustAbsoluteEnv(t, key)
+ info, err := os.Stat(value)
+ if err != nil || info.IsDir() || info.Mode()&0o111 == 0 {
+ t.Fatalf("%s must be executable: %v", key, err)
+ }
+ return value
+}
+
+func writeE2EJSON(t *testing.T, path string, value any) {
+ t.Helper()
+ if !filepath.IsAbs(path) {
+ t.Fatalf("result path must be absolute: %s", path)
+ }
+ raw, err := json.Marshal(value)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, raw, 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func consumeE2ECrashMarker(t *testing.T, path string) {
+ t.Helper()
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read one-shot crash marker: %v", err)
+ }
+ pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+ if err != nil || pid <= 1 {
+ t.Fatalf("invalid one-shot crash marker %q", raw)
+ }
+ if err := os.Remove(path); err != nil {
+ t.Fatalf("consume one-shot crash marker: %v", err)
+ }
+ if err := os.Unsetenv(linechainE2ECrashMarkerEnv); err != nil {
+ t.Fatalf("clear one-shot crash marker env: %v", err)
+ }
+}
+
+func TestLinechainE2EActiveHelper(t *testing.T) {
+ root := os.Getenv(linechainE2ERootEnv)
+ if root == "" {
+ return
+ }
+ if err := verifyManagedSingBox(root, "b", mustEnvPort(linechainE2EBPortEnv)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+type e2eDocument map[string]any
+
+type e2eSidecarPatch struct {
+ Schema, SourceLineUUID, SourceInboundTag string
+ ExpectedDownstreamLineUUID, DesiredDownstreamLineUUID *string
+}
+
+func (p e2eSidecarPatch) MarshalJSON() ([]byte, error) {
+ type wire struct {
+ Schema string `json:"schema"`
+ SourceLineUUID string `json:"source_line_uuid"`
+ SourceInboundTag string `json:"source_inbound_tag"`
+ Expected *string `json:"expected_downstream_line_uuid"`
+ Desired *string `json:"desired_downstream_line_uuid"`
+ }
+ return json.Marshal(wire{p.Schema, p.SourceLineUUID, p.SourceInboundTag, p.ExpectedDownstreamLineUUID, p.DesiredDownstreamLineUUID})
+}
+
+func bindE2EDocument(operation, basename, previousFragment string, fragment *string, sourceUUID, sourceTag string, expected, desired *string) e2eDocument {
+ patch := e2eSidecarPatch{"lattice.singbox-linechain-sidecar-patch.v1", sourceUUID, sourceTag, expected, desired}
+ patchBytes, _ := json.Marshal(patch)
+ var previous *string
+ if previousFragment != "" {
+ previous = &previousFragment
+ }
+ var fragmentSHA *string
+ if fragment != nil {
+ value := digestText(*fragment)
+ fragmentSHA = &value
+ }
+ patchSHA := digestBytes(patchBytes)
+ type artifact struct {
+ Schema string `json:"schema"`
+ Operation string `json:"operation"`
+ Basename string `json:"fragment_basename"`
+ Previous *string `json:"previous_fragment_sha256"`
+ Fragment *string `json:"fragment_sha256"`
+ Patch string `json:"sidecar_patch_sha256"`
+ }
+ artifactBytes, _ := json.Marshal(artifact{"lattice.singbox-linechain-artifact.v2", operation, basename, previous, fragmentSHA, patchSHA})
+ d := e2eDocument{"version": 2, "durable_protocol": "linechain-e3-v2", "operation": operation, "fragment_basename": basename, "fragment": fragment, "sidecar_patch": patch, "previous_fragment_sha256": previous, "fragment_sha256": fragmentSHA, "sidecar_patch_sha256": patchSHA, "artifact_sha256": digestBytes(artifactBytes)}
+ return d
+}
+
+func marshalE2EDocument(t *testing.T, d e2eDocument) []byte {
+ t.Helper()
+ b, err := json.Marshal(d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func canonicalJSONString(t *testing.T, raw string) string {
+ t.Helper()
+ var value any
+ if err := json.Unmarshal([]byte(raw), &value); err != nil {
+ t.Fatal(err)
+ }
+ b, err := json.Marshal(value)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(append(b, '\n'))
+}
+
+func openE2EManager(t *testing.T, bin, root, configDir, sidecarPath, txnDir string, bPort int) *linechain.Manager {
+ t.Helper()
+ for key, value := range map[string]string{
+ linechainE2EBinEnv: bin, linechainE2ERootEnv: root, linechainE2EConfigEnv: configDir,
+ linechainE2ESidecarEnv: sidecarPath, linechainE2EBPortEnv: strconv.Itoa(bPort),
+ } {
+ t.Setenv(key, value)
+ }
+ m, err := linechain.Open(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ configureE2EManager(t, m)
+ return m
+}
+
+func configureE2EManager(t *testing.T, m *linechain.Manager) {
+ t.Helper()
+ if err := m.ConfigureLayout(os.Getenv(linechainE2EConfigEnv), os.Getenv(linechainE2ESidecarEnv)); err != nil {
+ t.Fatal(err)
+ }
+ root := os.Getenv(linechainE2ERootEnv)
+ restart := []string{os.Args[0], "-test.run=^TestLinechainE2ERestartHelper$", "--", root}
+ verify := []string{os.Args[0], "-test.run=^TestLinechainE2EActiveHelper$", "--", root}
+ if err := m.ConfigureCommands(os.Getenv(linechainE2EBinEnv), restart, verify); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func applyAndResolve(t *testing.T, m *linechain.Manager, document []byte, taskID, leaseID string) {
+ t.Helper()
+ if err := m.Apply(context.Background(), bytes.NewReader(document), taskID, leaseID, digestText("e2e-direct:"+taskID)); err != nil {
+ t.Fatal(err)
+ }
+ result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0})
+ if err != nil || result.ExitCode != 0 || result.Error != "" {
+ t.Fatalf("resolve %s: result=%+v err=%v", taskID, result, err)
+ }
+ if err := m.Cleanup(taskID, leaseID); err != nil {
+ t.Fatalf("cleanup %s: %v", taskID, err)
+ }
+}
+
+func assertChainedInventory(t *testing.T, configDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA string, observerPort int) {
+ t.Helper()
+ inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", []string{filepath.Join(configDir, "config.json"), fragmentPath}, sidecarPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := findInventoryNode(t, inv, "source-b")
+ if n.OutboundRef != "chain-to-a" || n.OutboundServer != "127.0.0.1" || n.OutboundPort != strconv.Itoa(observerPort) || n.OutboundType != "vless" {
+ t.Fatalf("discovered outbound identity mismatch: %+v", n)
+ }
+ if n.LineUUID != lineUUIDB || n.DownstreamLineUUID != lineUUIDA {
+ t.Fatalf("discovered sidecar identity mismatch: %+v", n)
+ }
+}
+
+func assertUnchainedInventory(t *testing.T, configDir, sidecarPath string) {
+ t.Helper()
+ inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", []string{filepath.Join(configDir, "config.json")}, sidecarPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := findInventoryNode(t, inv, "source-b")
+ if n.OutboundRef != "" || n.OutboundServer != "" || n.OutboundPort != "" || n.DownstreamLineUUID != "" {
+ t.Fatalf("expected unchained inventory, got %+v", n)
+ }
+}
+
+func findInventoryNode(t *testing.T, inv model.SingBoxInventory, name string) model.SingBoxNode {
+ t.Helper()
+ for _, n := range inv.Nodes {
+ if n.Name == name {
+ return n
+ }
+ }
+ t.Fatalf("inventory lacks %q: %+v", name, inv.Nodes)
+ return model.SingBoxNode{}
+}
+
+func generateRealityKeypair(t *testing.T, bin string) (string, string) {
+ t.Helper()
+ out, err := exec.Command(bin, "generate", "reality-keypair").CombinedOutput()
+ if err != nil {
+ t.Fatalf("generate reality keypair: %v: %s", err, out)
+ }
+ var privateKey, publicKey string
+ for _, line := range strings.Split(string(out), "\n") {
+ if value, ok := strings.CutPrefix(line, "PrivateKey: "); ok {
+ privateKey = strings.TrimSpace(value)
+ }
+ if value, ok := strings.CutPrefix(line, "PublicKey: "); ok {
+ publicKey = strings.TrimSpace(value)
+ }
+ }
+ if privateKey == "" || publicKey == "" {
+ t.Fatalf("unexpected reality keypair output: %s", out)
+ }
+ return privateKey, publicKey
+}
+
+func requireSingBoxE2EBinary(t *testing.T) string {
+ t.Helper()
+ bin := os.Getenv(linechainE2EBinEnv)
+ if bin == "" || !filepath.IsAbs(bin) {
+ t.Fatalf("%s must name an absolute official sing-box 1.13.x binary", linechainE2EBinEnv)
+ }
+ out, err := exec.Command(bin, "version").CombinedOutput()
+ if err != nil || !strings.Contains(string(out), "sing-box version 1.13.") {
+ t.Fatalf("%s is not sing-box 1.13.x: %v\n%s", bin, err, out)
+ }
+ return bin
+}
+
+func mustMkdir(t *testing.T, path string) {
+ t.Helper()
+ if err := os.Mkdir(path, 0o700); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func writeFile(t *testing.T, path, body string) {
+ t.Helper()
+ if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func freePort(t *testing.T) int {
+ t.Helper()
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ return l.Addr().(*net.TCPAddr).Port
+}
+
+func startEchoOrigin(t *testing.T) *net.TCPAddr {
+ t.Helper()
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = l.Close() })
+ go func() {
+ for {
+ c, err := l.Accept()
+ if err != nil {
+ return
+ }
+ go func() { defer c.Close(); _, _ = io.Copy(c, c) }()
+ }
+ }()
+ return l.Addr().(*net.TCPAddr)
+}
+
+type tcpObserver struct {
+ listener net.Listener
+ count chan struct{}
+}
+
+func startTCPObserver(t *testing.T, target string) *tcpObserver {
+ t.Helper()
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ o := &tcpObserver{listener: l, count: make(chan struct{}, 128)}
+ t.Cleanup(func() { _ = l.Close() })
+ go func() {
+ for {
+ incoming, err := l.Accept()
+ if err != nil {
+ return
+ }
+ o.count <- struct{}{}
+ go func() {
+ defer incoming.Close()
+ upstream, err := net.Dial("tcp", target)
+ if err != nil {
+ return
+ }
+ defer upstream.Close()
+ go func() { _, _ = io.Copy(upstream, incoming); _ = upstream.(*net.TCPConn).CloseWrite() }()
+ _, _ = io.Copy(incoming, upstream)
+ }()
+ }
+ }()
+ return o
+}
+
+func (o *tcpObserver) Port() int { return o.listener.Addr().(*net.TCPAddr).Port }
+func (o *tcpObserver) Count() int {
+ n := 0
+ for {
+ select {
+ case <-o.count:
+ n++
+ default:
+ return n
+ }
+ }
+}
+
+func startManagedSingBox(t *testing.T, bin, root, name, configDir string, port int) {
+ t.Helper()
+ if err := restartManagedSingBox(bin, root, name, configDir, port); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := killManagedSingBox(root, name); err != nil {
+ t.Errorf("cleanup managed sing-box %s: %v", name, err)
+ }
+ })
+}
+
+func restartManagedSingBox(bin, root, name, configDir string, port int) error {
+ if err := killManagedSingBox(root, name); err != nil {
+ return err
+ }
+ logPath := filepath.Join(root, name+".log")
+ logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+ if err != nil {
+ return err
+ }
+ cmd := exec.Command(bin, "run", "-C", configDir)
+ cmd.Stdout, cmd.Stderr = logFile, logFile
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ if err := cmd.Start(); err != nil {
+ _ = logFile.Close()
+ return err
+ }
+ done := make(chan error, 1)
+ managedE2EProcesses.Lock()
+ managedE2EProcesses.done[cmd.Process.Pid] = done
+ managedE2EProcesses.Unlock()
+ go func() { done <- cmd.Wait() }()
+ _ = logFile.Close()
+ if err := os.WriteFile(filepath.Join(root, name+".pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil {
+ _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+ leaderReaped := false
+ exited := waitManagedProcess(cmd.Process.Pid, done, &leaderReaped, 2*time.Second)
+ managedE2EProcesses.Lock()
+ delete(managedE2EProcesses.done, cmd.Process.Pid)
+ managedE2EProcesses.Unlock()
+ if !exited {
+ return fmt.Errorf("write %s pid file: %w; process group %d remained after SIGKILL", name, err, cmd.Process.Pid)
+ }
+ return err
+ }
+ if err := waitForPort(port, 8*time.Second); err != nil {
+ cleanupErr := killManagedSingBox(root, name)
+ logBytes, _ := os.ReadFile(logPath)
+ if cleanupErr != nil {
+ return fmt.Errorf("start %s: %w: %s; cleanup: %v", name, err, logBytes, cleanupErr)
+ }
+ return fmt.Errorf("start %s: %w: %s", name, err, logBytes)
+ }
+ return nil
+}
+
+func killManagedSingBox(root, name string) (retErr error) {
+ pidPath := filepath.Join(root, name+".pid")
+ raw, err := os.ReadFile(pidPath)
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+ if err != nil || pid <= 1 {
+ parseErr := fmt.Errorf("invalid %s pid %q", name, raw)
+ if removeErr := os.Remove(pidPath); removeErr != nil && !os.IsNotExist(removeErr) {
+ return fmt.Errorf("%v; remove invalid pid file: %w", parseErr, removeErr)
+ }
+ return parseErr
+ }
+ managedE2EProcesses.Lock()
+ done := managedE2EProcesses.done[pid]
+ managedE2EProcesses.Unlock()
+ defer func() {
+ managedE2EProcesses.Lock()
+ delete(managedE2EProcesses.done, pid)
+ managedE2EProcesses.Unlock()
+ if err := os.Remove(pidPath); err != nil && !os.IsNotExist(err) {
+ if retErr == nil {
+ retErr = fmt.Errorf("remove %s pid file: %w", name, err)
+ } else {
+ retErr = fmt.Errorf("%v; remove %s pid file: %w", retErr, name, err)
+ }
+ }
+ }()
+ if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH {
+ return err
+ }
+ leaderReaped := done == nil
+ if waitManagedProcess(pid, done, &leaderReaped, 2*time.Second) {
+ return nil
+ }
+ if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
+ return err
+ }
+ if waitManagedProcess(pid, done, &leaderReaped, 2*time.Second) {
+ return nil
+ }
+ return fmt.Errorf("%s process %d remained after SIGKILL", name, pid)
+}
+
+func waitManagedProcess(pid int, done <-chan error, leaderReaped *bool, timeout time.Duration) bool {
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if !*leaderReaped && done != nil {
+ select {
+ case <-done:
+ *leaderReaped = true
+ default:
+ }
+ }
+ if *leaderReaped && syscall.Kill(-pid, 0) == syscall.ESRCH {
+ return true
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ return false
+}
+
+func TestKillManagedSingBoxTerminatesProcessGroupAndClearsState(t *testing.T) {
+ root := t.TempDir()
+ childPIDPath := filepath.Join(root, "child.pid")
+ cmd := exec.Command("sh", "-c", `trap 'exit 0' TERM; sh -c 'trap "" TERM; while :; do sleep 1; done' & echo $! > "$1"; wait`, "managed-test", childPIDPath)
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+ done := make(chan error, 1)
+ managedE2EProcesses.Lock()
+ managedE2EProcesses.done[cmd.Process.Pid] = done
+ managedE2EProcesses.Unlock()
+ go func() { done <- cmd.Wait() }()
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ if _, err := os.Stat(childPIDPath); err == nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("TERM-ignoring child did not start")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ pidPath := filepath.Join(root, "test.pid")
+ if err := os.WriteFile(pidPath, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := killManagedSingBox(root, "test"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(pidPath); !os.IsNotExist(err) {
+ t.Fatalf("pid file survived cleanup: %v", err)
+ }
+ managedE2EProcesses.Lock()
+ _, tracked := managedE2EProcesses.done[cmd.Process.Pid]
+ managedE2EProcesses.Unlock()
+ if tracked {
+ t.Fatal("managed process survived in reap map")
+ }
+ if err := syscall.Kill(-cmd.Process.Pid, 0); err != syscall.ESRCH {
+ t.Fatalf("managed process group survived cleanup: %v", err)
+ }
+}
+
+func TestKillManagedSingBoxRejectsAndRemovesInvalidPID(t *testing.T) {
+ root := t.TempDir()
+ pidPath := filepath.Join(root, "invalid.pid")
+ if err := os.WriteFile(pidPath, []byte("not-a-pid"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := killManagedSingBox(root, "invalid"); err == nil || !strings.Contains(err.Error(), "invalid invalid pid") {
+ t.Fatalf("invalid pid error = %v", err)
+ }
+ if _, err := os.Stat(pidPath); !os.IsNotExist(err) {
+ t.Fatalf("invalid pid file survived cleanup: %v", err)
+ }
+}
+
+func killMarkerProcess(t *testing.T, marker string) {
+ t.Helper()
+ deadline := time.Now().Add(3 * time.Second)
+ for time.Now().Before(deadline) {
+ raw, err := os.ReadFile(marker)
+ if os.IsNotExist(err) {
+ return
+ }
+ if err != nil {
+ t.Errorf("read helper marker: %v", err)
+ return
+ }
+ pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+ if err != nil || pid <= 1 {
+ t.Errorf("invalid helper marker %q", raw)
+ return
+ }
+ _ = syscall.Kill(-pid, syscall.SIGKILL)
+ _ = syscall.Kill(pid, syscall.SIGKILL)
+ if err := syscall.Kill(pid, 0); err == syscall.ESRCH {
+ _ = os.Remove(marker)
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Errorf("restart helper from %s remained after cleanup", marker)
+}
+
+func verifyManagedSingBox(root, name string, port int) error {
+ raw, err := os.ReadFile(filepath.Join(root, name+".pid"))
+ if err != nil {
+ return err
+ }
+ pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+ if err != nil || pid <= 1 {
+ return fmt.Errorf("invalid pid %q", raw)
+ }
+ if err := syscall.Kill(pid, 0); err != nil {
+ return fmt.Errorf("sing-box process inactive: %w", err)
+ }
+ return waitForPort(port, time.Second)
+}
+
+func waitForPort(port int, timeout time.Duration) error {
+ deadline := time.Now().Add(timeout)
+ address := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
+ for time.Now().Before(deadline) {
+ c, err := net.DialTimeout("tcp", address, 50*time.Millisecond)
+ if err == nil {
+ _ = c.Close()
+ return nil
+ }
+ time.Sleep(25 * time.Millisecond)
+ }
+ return fmt.Errorf("port %d did not become ready", port)
+}
+
+func waitForFileOrProcess(t *testing.T, path string, done <-chan error, output *bytes.Buffer) {
+ t.Helper()
+ deadline := time.Now().Add(8 * time.Second)
+ for time.Now().Before(deadline) {
+ if _, err := os.Stat(path); err == nil {
+ return
+ }
+ select {
+ case err := <-done:
+ t.Fatalf("apply helper exited before crash marker: %v: %s", err, output.String())
+ default:
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ t.Fatalf("timed out waiting for helper signal %s: %s", path, output.String())
+}
+
+func waitForExactPair(t *testing.T, fragmentPath, fragment, sidecarPath, sidecar string) {
+ t.Helper()
+ deadline := time.Now().Add(8 * time.Second)
+ for time.Now().Before(deadline) {
+ f, ferr := os.ReadFile(fragmentPath)
+ s, serr := os.ReadFile(sidecarPath)
+ if ferr == nil && serr == nil && string(f) == fragment && string(s) == sidecar {
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ f, ferr := os.ReadFile(fragmentPath)
+ s, serr := os.ReadFile(sidecarPath)
+ t.Fatalf("apply helper did not publish exact pair: fragment err=%v got=%q want=%q; sidecar err=%v got=%q want=%q", ferr, f, fragment, serr, s, sidecar)
+}
+
+func mustEnvPort(key string) int {
+ port, err := strconv.Atoi(os.Getenv(key))
+ if err != nil || port < 1 || port > 65535 {
+ panic("invalid " + key)
+ }
+ return port
+}
+
+func digestText(s string) string { return digestBytes([]byte(s)) }
+func digestPointer(s *string) string {
+ if s == nil {
+ return ""
+ }
+ return digestText(*s)
+}
+func digestBytes(b []byte) string {
+ sum := sha256.Sum256(b)
+ return hex.EncodeToString(sum[:])
+}
+
+func assertSOCKSEcho(t *testing.T, socksPort int, target *net.TCPAddr) {
+ t.Helper()
+ c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", socksPort), 2*time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer c.Close()
+ _ = c.SetDeadline(time.Now().Add(5 * time.Second))
+ if _, err = c.Write([]byte{5, 1, 0}); err != nil {
+ t.Fatal(err)
+ }
+ reply := make([]byte, 2)
+ if _, err = io.ReadFull(c, reply); err != nil || !bytes.Equal(reply, []byte{5, 0}) {
+ t.Fatalf("SOCKS greeting: %v %v", reply, err)
+ }
+ req := []byte{5, 1, 0, 1, 127, 0, 0, 1, 0, 0}
+ binary.BigEndian.PutUint16(req[8:], uint16(target.Port))
+ if _, err = c.Write(req); err != nil {
+ t.Fatal(err)
+ }
+ head := make([]byte, 4)
+ if _, err = io.ReadFull(c, head); err != nil || head[1] != 0 {
+ t.Fatalf("SOCKS connect: %v %v", head, err)
+ }
+ n := 6
+ if head[3] == 3 {
+ one := make([]byte, 1)
+ if _, err = io.ReadFull(c, one); err != nil {
+ t.Fatal(err)
+ }
+ n = int(one[0]) + 2
+ } else if head[3] == 4 {
+ n = 18
+ }
+ if _, err = io.ReadFull(c, make([]byte, n)); err != nil {
+ t.Fatal(err)
+ }
+ payload := []byte("lattice-linechain-e2e")
+ if _, err = c.Write(payload); err != nil {
+ t.Fatal(err)
+ }
+ got := make([]byte, 0, 128)
+ buf := make([]byte, 32)
+ for len(got) < cap(got) && !bytes.Contains(got, payload) {
+ n, readErr := c.Read(buf)
+ got = append(got, buf[:n]...)
+ if readErr != nil {
+ err = readErr
+ break
+ }
+ }
+ if !bytes.Contains(got, payload) {
+ t.Fatalf("chain echo = %q err=%v", got, err)
+ }
+}
diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go
index 635f676..673da3d 100644
--- a/cmd/lattice-agent/main.go
+++ b/cmd/lattice-agent/main.go
@@ -4,6 +4,8 @@ import (
"bytes"
"context"
"crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
"encoding/json"
"errors"
"flag"
@@ -27,6 +29,7 @@ import (
"github.com/LatticeNet/lattice-node-agent/internal/guardreality"
"github.com/LatticeNet/lattice-node-agent/internal/hostfacts"
"github.com/LatticeNet/lattice-node-agent/internal/ipdiscover"
+ "github.com/LatticeNet/lattice-node-agent/internal/linechain"
"github.com/LatticeNet/lattice-node-agent/internal/metrics"
"github.com/LatticeNet/lattice-node-agent/internal/prober"
"github.com/LatticeNet/lattice-node-agent/internal/proxyusage"
@@ -37,10 +40,10 @@ import (
"github.com/LatticeNet/lattice-sdk/model"
)
-var version = "0.3.3"
-var compatServerMin = "v0.2.2-alpha.2"
+var version = "0.3.4-alpha.1"
+var compatServerMin = "v0.2.2-alpha.19"
var compatDashboardMin = "v0.2.2-alpha.7"
-var compatChannel = "stable"
+var compatChannel = "alpha"
type agentCompatibility struct {
ServerMin string `json:"server_min"`
@@ -62,15 +65,23 @@ func compatibilityPayload() agentCompatibility {
var httpClient = &http.Client{Timeout: 30 * time.Second}
const (
- defaultDebugMaxLineBytes = 4096
- defaultDebugMaxBatchLines = 100
- debugSinkMaxLines = 1000
- guardRealityReportTimeout = 10 * time.Second
- guardManagedSHACapability = "netguard-managed-sha-v1"
- agentCapabilitiesHeader = "X-Lattice-Agent-Capabilities"
+ defaultDebugMaxLineBytes = 4096
+ defaultDebugMaxBatchLines = 100
+ debugSinkMaxLines = 1000
+ guardRealityReportTimeout = 10 * time.Second
+ guardManagedSHACapability = "netguard-managed-sha-v1"
+ durableTaskResultCapability = "durable-task-result-v1"
+ agentCapabilitiesHeader = "X-Lattice-Agent-Capabilities"
)
func reportedCapabilities() []string {
+ return []string{durableTaskResultCapability, guardManagedSHACapability}
+}
+
+func capabilitiesFor(linechainReady bool) []string {
+ if linechainReady {
+ return reportedCapabilities()
+ }
return []string{guardManagedSHACapability}
}
@@ -154,6 +165,8 @@ type agentConfig struct {
SingBoxMeta string
LogStateDir string
TaskOutboxDir string
+ LinechainTxnDir string
+ LinechainReady bool
}
type agentRuntimePayload struct {
@@ -189,6 +202,7 @@ func main() {
var printVersion bool
var printCompat bool
var printGuardManagedSHA bool
+ var applyLinechain bool
flag.StringVar(&cfg.Server, "server", env("LATTICE_SERVER", "http://127.0.0.1:8088"), "server base URL")
flag.StringVar(&cfg.NodeID, "node-id", os.Getenv("LATTICE_NODE_ID"), "node id")
flag.StringVar(&cfg.Token, "token", os.Getenv("LATTICE_NODE_TOKEN"), "node enrollment token")
@@ -247,6 +261,8 @@ func main() {
flag.StringVar(&cfg.SingBoxMeta, "singbox-meta", env("LATTICE_SINGBOX_META", ""), "design-15 sing-box sidecar metadata path for -singbox-discover (default /etc/sing-box/lattice-metadata.json)")
flag.StringVar(&cfg.LogStateDir, "log-state-dir", os.Getenv("LATTICE_LOG_STATE_DIR"), "directory for log-tail checkpoints (empty disables checkpoint persistence; sources still tail from end)")
flag.StringVar(&cfg.TaskOutboxDir, "task-outbox-dir", os.Getenv("LATTICE_TASK_OUTBOX_DIR"), "base directory for durable task-result journals (default: log state dir, or the user cache directory for manual runs)")
+ flag.StringVar(&cfg.LinechainTxnDir, "linechain-txn-dir", os.Getenv("LATTICE_LINECHAIN_TXN_DIR"), "private directory for crash-recoverable linechain transactions")
+ flag.BoolVar(&applyLinechain, "linechain-apply", false, "apply one bounded linechain document from stdin and exit")
flag.BoolVar(&printVersion, "version", false, "print lattice-agent version and exit")
flag.BoolVar(&printCompat, "compat-json", false, "print embedded server/dashboard compatibility metadata and exit")
flag.BoolVar(&printGuardManagedSHA, "guard-managed-sha", false, "print the canonical SHA-256 of the managed lattice_guard nft table and exit")
@@ -267,6 +283,24 @@ func main() {
}
return
}
+ if applyLinechain {
+ dir, err := linechainTransactionDir(cfg)
+ if err != nil {
+ log.Fatalf("linechain transaction path failed: %v", err)
+ }
+ manager, err := linechain.OpenHelper(dir)
+ if err != nil {
+ log.Fatalf("linechain transaction manager failed: %v", err)
+ }
+ defer manager.Close()
+ if err := manager.ConfigureLayout(os.Getenv("LATTICE_LINECHAIN_CONFIG_DIR"), os.Getenv("LATTICE_LINECHAIN_SIDECAR_PATH")); err != nil {
+ log.Fatalf("linechain local layout failed: %v", err)
+ }
+ if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID"), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil {
+ log.Fatalf("linechain apply failed: %v", err)
+ }
+ return
+ }
cfg.Debug = cfg.LocalDebug
cfg.DebugMaxLineBytes = defaultDebugMaxLineBytes
cfg.DebugMaxBatchLines = defaultDebugMaxBatchLines
@@ -331,6 +365,31 @@ func main() {
log.Fatalf("task result outbox initialization failed: %v", err)
}
defer taskResults.Close()
+ linechainDir, err := linechainTransactionDir(cfg)
+ if err != nil {
+ log.Fatalf("linechain transaction path failed: %v", err)
+ }
+ linechainManager, err := linechain.Open(linechainDir)
+ if err != nil {
+ log.Fatalf("linechain transaction manager initialization failed: %v", err)
+ }
+ defer linechainManager.Close()
+ linechainConfigDir, linechainSidecarPath, layoutErr := singboxdiscover.ResolveRuntimeLayout(cfg.SingBoxMeta)
+ if layoutErr != nil {
+ log.Printf("warning: durable linechain tasks disabled: %v", layoutErr)
+ } else if err := linechainManager.ConfigureLayout(linechainConfigDir, linechainSidecarPath); err != nil {
+ log.Printf("warning: durable linechain tasks disabled: %v", err)
+ } else {
+ cfg.LinechainReady = true
+ }
+ for {
+ if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err == nil {
+ break
+ } else {
+ log.Printf("linechain recovery blocked readiness: %v", err)
+ time.Sleep(cfg.Interval)
+ }
+ }
agentBinary, err := os.Executable()
if err != nil {
log.Fatalf("resolve lattice-agent executable failed: %v", err)
@@ -353,7 +412,7 @@ func main() {
if err := postAgentJSON(cfg, "/api/agent/hello", map[string]any{
"version": version,
"compatibility": compatibilityPayload(),
- "capabilities": reportedCapabilities(),
+ "capabilities": capabilitiesFor(cfg.LinechainReady),
"public_ip": cfg.PublicIP,
"public_ipv6": cfg.PublicIPv6,
"internal_ip": cfg.InternalIP,
@@ -381,13 +440,18 @@ func main() {
runner := taskexec.Runner{
AllowExec: cfg.AllowExec, AllowRoot: cfg.AllowRoot, Cgroup: cfg.taskCgroupConfig(),
- WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary,
+ WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary, LinechainTxnDir: linechainDir, LinechainConfigDir: linechainConfigDir, LinechainSidecarPath: linechainSidecarPath,
}
monitors := newMonitorManager(cfg)
logTailers := newLogTailManager(cfg)
ticker := time.NewTicker(cfg.Interval)
defer ticker.Stop()
for {
+ if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil {
+ log.Printf("linechain recovery blocked cycle: %v", err)
+ <-ticker.C
+ continue
+ }
if agentCfg, err := fetchAgentConfig(cfg); err != nil {
debugf(cfg, "agent config fetch failed: %v", err)
} else {
@@ -405,7 +469,7 @@ func main() {
if err := reportSingBoxInventory(cfg); err != nil {
log.Printf("singbox discover error: %v", err)
}
- if err := runTasks(cfg, runner, taskResults); err != nil {
+ if err := runTasks(cfg, runner, taskResults, linechainManager); err != nil {
log.Printf("task poll error: %v", err)
}
if assigned, err := fetchMonitors(cfg); err != nil {
@@ -677,7 +741,7 @@ func reportMetrics(cfg agentConfig) error {
return postAgentJSON(cfg, "/api/agent/metrics", map[string]any{
"version": version,
"compatibility": compatibilityPayload(),
- "capabilities": reportedCapabilities(),
+ "capabilities": capabilitiesFor(cfg.LinechainReady),
"agent_runtime": agentRuntimePayload{
AllowExec: cfg.AllowExec,
AllowRootExec: cfg.AllowRoot,
@@ -1113,12 +1177,26 @@ type taskResultOutbox interface {
Remove(taskoutbox.Entry) error
}
+type taskResultOutboxSnapshot interface {
+ Snapshot() ([]taskoutbox.Entry, error)
+}
+
type leasedAgentTask struct {
model.Task
- DurableResult bool `json:"durable_result"`
+ DurableResult bool `json:"durable_result"`
+ DurableProtocol string `json:"durable_protocol,omitempty"`
}
-func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error {
+func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, managers ...*linechain.Manager) error {
+ var manager *linechain.Manager
+ if len(managers) > 0 {
+ manager = managers[0]
+ }
+ if manager != nil {
+ if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil {
+ return err
+ }
+ }
if err := outbox.RecoverInterrupted(cfg.NodeID); err != nil {
return fmt.Errorf("recover interrupted task results: %w", err)
}
@@ -1130,7 +1208,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
return err
}
req.Header.Set("Authorization", "Bearer "+cfg.Token)
- req.Header.Set(agentCapabilitiesHeader, strings.Join(reportedCapabilities(), ","))
+ req.Header.Set(agentCapabilitiesHeader, strings.Join(capabilitiesFor(cfg.LinechainReady), ","))
resp, err := httpClient.Do(req)
if err != nil {
return err
@@ -1146,6 +1224,12 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
debugf(cfg, "tasks fetched: count=%d", len(tasks))
for _, leased := range tasks {
task := leased.Task
+ if err := validateDurablePair(leased, manager, cfg.LinechainReady); err != nil {
+ return err
+ }
+ if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v2" && leased.DurableProtocol != "netguard-v1" {
+ return fmt.Errorf("unsupported durable protocol %q", leased.DurableProtocol)
+ }
if !leased.DurableResult {
debugf(cfg, "task start without durable-result protocol: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec)
result := runner.Run(task)
@@ -1158,7 +1242,15 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
}
continue
}
- committed, err := outbox.Begin(task)
+ var committed bool
+ var err error
+ if typed, ok := outbox.(interface {
+ BeginWithProtocol(model.Task, string) (bool, error)
+ }); ok {
+ committed, err = typed.BeginWithProtocol(task, leased.DurableProtocol)
+ } else {
+ committed, err = outbox.Begin(task)
+ }
if err != nil {
journalErr := fmt.Errorf("journal task lease %s before execution: %w", task.ID, err)
if committed {
@@ -1189,8 +1281,25 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
continue
}
debugf(cfg, "task start: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec)
- result := runner.Run(task)
+ var result model.TaskResult
+ if isLinechainTask(leased) {
+ typed, ok := runner.(interface {
+ RunLinechain(model.Task) model.TaskResult
+ })
+ if !ok {
+ return fmt.Errorf("linechain task runner does not expose trusted E3 execution")
+ }
+ result = typed.RunLinechain(task)
+ } else {
+ result = runner.Run(task)
+ }
result.NodeID = cfg.NodeID
+ if manager != nil && isLinechainTask(leased) {
+ result, err = manager.ResolveAfterRun(context.Background(), task, result)
+ if err != nil {
+ return fmt.Errorf("resolve linechain task %s: %w", task.ID, err)
+ }
+ }
debugf(cfg, "task complete: id=%s exit_code=%d error=%t", task.ID, result.ExitCode, result.Error != "")
completed, completeErr := outbox.Complete(result)
if completeErr != nil {
@@ -1203,12 +1312,28 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
if confirmErr := outbox.ConfirmDurability(); confirmErr != nil {
return fmt.Errorf("%v; confirm published task result: %w", journalErr, confirmErr)
}
+ if manager != nil && isLinechainTask(leased) {
+ if cleanupErr := manager.Cleanup(task.ID, task.LeaseID); cleanupErr != nil {
+ if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil {
+ return fmt.Errorf("%v; cleanup confirmed linechain task: %v; upload stable result: %w", journalErr, cleanupErr, uploadErr)
+ }
+ return fmt.Errorf("%v; cleanup confirmed linechain task: %w; result remains replayable", journalErr, cleanupErr)
+ }
+ }
if flushErr := flushTaskResults(cfg, outbox); flushErr != nil {
return fmt.Errorf("%v; upload confirmed task result: %w", journalErr, flushErr)
}
}
return journalErr
}
+ if manager != nil && isLinechainTask(leased) {
+ if err := manager.Cleanup(task.ID, task.LeaseID); err != nil {
+ if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil {
+ return fmt.Errorf("cleanup linechain task %s: %v; upload stable result: %w", task.ID, err, uploadErr)
+ }
+ return fmt.Errorf("cleanup linechain task %s after durable outbox completion; result remains replayable: %w", task.ID, err)
+ }
+ }
if err := flushTaskResults(cfg, outbox); err != nil {
return err
}
@@ -1216,7 +1341,198 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error
return nil
}
+func crossCheckLinechainAuthority(manager *linechain.Manager, outbox taskResultOutbox) error {
+ _, err := captureLinechainAuthority(manager, outbox)
+ return err
+}
+
+func captureLinechainAuthority(manager *linechain.Manager, outbox taskResultOutbox) (map[string]linechain.RecoveryAuthority, error) {
+ typed, ok := outbox.(taskResultOutboxSnapshot)
+ if !ok {
+ return nil, nil
+ }
+ entries, err := typed.Snapshot()
+ if err != nil {
+ return nil, err
+ }
+ refs, err := manager.Snapshot()
+ if err != nil {
+ return nil, err
+ }
+ journals := map[string]linechain.JournalRef{}
+ for _, ref := range refs {
+ journals[ref.TaskID+"\x00"+ref.LeaseID] = ref
+ }
+ matchedE3 := map[string]struct{}{}
+ for _, entry := range entries {
+ key := entry.Task.ID + "\x00" + entry.Task.LeaseID
+ ref, hasJournal := journals[key]
+ if hasJournal && entry.DurableProtocol != "linechain-e3-v2" {
+ return nil, fmt.Errorf("linechain journal %s has mismatched outbox protocol", entry.Task.ID)
+ }
+ if hasJournal && entry.DurableProtocol == "linechain-e3-v2" {
+ if ref.TaskScriptSHA != linechainTaskScriptSHA(entry.Task.Script) {
+ return nil, fmt.Errorf("linechain journal %s does not match the exact outbox task script", entry.Task.ID)
+ }
+ artifactSHA, err := linechainArtifactSHAFromTaskScript(entry.Task.Script)
+ if err != nil || artifactSHA != ref.ArtifactSHA256 {
+ return nil, fmt.Errorf("linechain journal %s does not match the issued artifact in the exact outbox script", entry.Task.ID)
+ }
+ matchedE3[key] = struct{}{}
+ }
+ if entry.State == "leased" && entry.DurableProtocol == "linechain-e3-v2" {
+ if !hasJournal {
+ return nil, fmt.Errorf("leased E3 outbox %s lacks exact linechain journal", entry.Task.ID)
+ }
+ }
+ if entry.State == "completed" && entry.DurableProtocol == "linechain-e3-v2" && hasJournal {
+ if !ref.Terminal || entry.Result == nil || ref.Result == nil || !sameTaskResult(*entry.Result, *ref.Result) {
+ return nil, fmt.Errorf("completed E3 outbox %s conflicts with linechain terminal result", entry.Task.ID)
+ }
+ }
+ }
+ for key := range journals {
+ if _, ok := matchedE3[key]; !ok {
+ return nil, fmt.Errorf("linechain journal lacks exact leased E3 outbox: %q", key)
+ }
+ }
+ authority := make(map[string]linechain.RecoveryAuthority, len(journals))
+ for key, ref := range journals {
+ authority[key] = linechain.RecoveryAuthority{TaskScriptSHA: ref.TaskScriptSHA, ArtifactSHA256: ref.ArtifactSHA256, Phase: ref.Phase, ResultSHA256: taskResultPointerSHA(ref.Result), JournalSHA256: ref.JournalSHA256}
+ }
+ return authority, nil
+}
+
+func sameTaskResult(a, b model.TaskResult) bool {
+ aBytes, aErr := json.Marshal(a)
+ bBytes, bErr := json.Marshal(b)
+ return aErr == nil && bErr == nil && bytes.Equal(aBytes, bBytes)
+}
+
+func linechainTaskScriptSHA(script string) string {
+ sum := sha256.Sum256([]byte(script))
+ return hex.EncodeToString(sum[:])
+}
+
+func linechainArtifactSHAFromTaskScript(script string) (string, error) {
+ const prefix = "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '"
+ const suffix = "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n"
+ if !strings.HasPrefix(script, prefix) || !strings.HasSuffix(script, suffix) {
+ return "", fmt.Errorf("linechain task script is not the canonical v2 wrapper")
+ }
+ encoded := strings.TrimSuffix(strings.TrimPrefix(script, prefix), suffix)
+ raw, err := base64.StdEncoding.DecodeString(encoded)
+ if err != nil {
+ return "", fmt.Errorf("decode linechain task document: %w", err)
+ }
+ fields, err := decodeUniqueTopLevelFields(raw)
+ if err != nil {
+ return "", fmt.Errorf("decode linechain task document: %w", err)
+ }
+ var version int
+ var protocol, artifact string
+ if err := json.Unmarshal(fields["version"], &version); err != nil || version != 2 {
+ return "", fmt.Errorf("linechain task document version is invalid")
+ }
+ if err := json.Unmarshal(fields["durable_protocol"], &protocol); err != nil || protocol != "linechain-e3-v2" {
+ return "", fmt.Errorf("linechain task document protocol is invalid")
+ }
+ if err := json.Unmarshal(fields["artifact_sha256"], &artifact); err != nil || !guardManagedSHARe.MatchString(artifact) {
+ return "", fmt.Errorf("linechain task document artifact is invalid")
+ }
+ return artifact, nil
+}
+
+func decodeUniqueTopLevelFields(raw []byte) (map[string]json.RawMessage, error) {
+ dec := json.NewDecoder(bytes.NewReader(raw))
+ token, err := dec.Token()
+ if err != nil || token != json.Delim('{') {
+ return nil, fmt.Errorf("document must be an object")
+ }
+ fields := make(map[string]json.RawMessage)
+ for dec.More() {
+ token, err := dec.Token()
+ if err != nil {
+ return nil, err
+ }
+ name, ok := token.(string)
+ if !ok {
+ return nil, fmt.Errorf("document field name is invalid")
+ }
+ if _, exists := fields[name]; exists {
+ return nil, fmt.Errorf("duplicate document field %s", name)
+ }
+ var value json.RawMessage
+ if err := dec.Decode(&value); err != nil {
+ return nil, err
+ }
+ fields[name] = value
+ }
+ if _, err := dec.Token(); err != nil {
+ return nil, err
+ }
+ if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) {
+ return nil, fmt.Errorf("unexpected trailing data")
+ }
+ return fields, nil
+}
+
+func taskResultPointerSHA(result *model.TaskResult) string {
+ if result == nil {
+ return ""
+ }
+ b, err := json.Marshal(result)
+ if err != nil {
+ return ""
+ }
+ sum := sha256.Sum256(b)
+ return hex.EncodeToString(sum[:])
+}
+
+func validateDurablePair(task leasedAgentTask, manager *linechain.Manager, ready bool) error {
+ if !task.DurableResult {
+ if task.DurableProtocol != "" {
+ return fmt.Errorf("non-durable task has protocol %q", task.DurableProtocol)
+ }
+ return nil
+ }
+ if task.DurableProtocol != "linechain-e3-v2" && task.DurableProtocol != "netguard-v1" {
+ return fmt.Errorf("durable task has unsupported protocol %q", task.DurableProtocol)
+ }
+ if task.DurableProtocol == "linechain-e3-v2" && (manager == nil || !ready) {
+ return fmt.Errorf("linechain task received while durable linechain is unavailable")
+ }
+ return nil
+}
+
+func isLinechainTask(task leasedAgentTask) bool {
+ return task.DurableResult && task.DurableProtocol == "linechain-e3-v2"
+}
+
+func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error {
+ authority, err := captureLinechainAuthority(manager, outbox)
+ if err != nil {
+ return err
+ }
+ return manager.RequireRecoveredAuthorized(ctx, func(result model.TaskResult) error {
+ committed, err := outbox.Complete(result)
+ if err == nil {
+ return nil
+ }
+ if committed {
+ return outbox.ConfirmDurability()
+ }
+ // An exact completed outbox may remain after a crash before transaction
+ // cleanup. Complete implementations treat that replay idempotently.
+ return err
+ }, nodeID, authority)
+}
+
func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error {
+ return flushTaskResultsRetain(cfg, outbox, false)
+}
+
+func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain bool) error {
pending, err := outbox.Pending()
if err != nil {
return fmt.Errorf("read pending task results: %w", err)
@@ -1230,6 +1546,12 @@ func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error {
}, nil); err != nil {
return fmt.Errorf("flush durable task result %s: %w", entry.Task.ID, err)
}
+ // Pending entries predate typed lease metadata; retain only the exact
+ // helper namespace marker during cleanup recovery. Live classification is
+ // always driven by durable_protocol on the leased response.
+ if retain && entry.DurableProtocol == "linechain-e3-v2" {
+ continue
+ }
if err := outbox.Remove(entry); err != nil {
return fmt.Errorf("remove acknowledged task result %s: %w", entry.Task.ID, err)
}
@@ -1253,6 +1575,26 @@ func taskResultOutboxDir(cfg agentConfig) (string, error) {
return filepath.Join(base, "task-outbox", fmt.Sprintf("%x", nodeHash[:])), nil
}
+func linechainTransactionDir(cfg agentConfig) (string, error) {
+ dir := strings.TrimSpace(cfg.LinechainTxnDir)
+ if dir == "" {
+ base := strings.TrimSpace(cfg.LogStateDir)
+ if base == "" {
+ cacheDir, err := os.UserCacheDir()
+ if err != nil {
+ return "", err
+ }
+ base = filepath.Join(cacheDir, "lattice-agent")
+ }
+ dir = filepath.Join(base, "linechain-txn")
+ }
+ dir = filepath.Clean(dir)
+ if !filepath.IsAbs(dir) || dir == string(filepath.Separator) {
+ return "", fmt.Errorf("linechain transaction directory must be absolute and non-root")
+ }
+ return dir, nil
+}
+
func postAgentJSON(cfg agentConfig, path string, payload map[string]any, out any) error {
return postAgentJSONContext(context.Background(), cfg, path, payload, out)
}
diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go
index c8dcbfc..398acd8 100644
--- a/cmd/lattice-agent/main_test.go
+++ b/cmd/lattice-agent/main_test.go
@@ -3,6 +3,7 @@ package main
import (
"bytes"
"context"
+ "encoding/base64"
"encoding/json"
"errors"
"io"
@@ -14,12 +15,21 @@ import (
"testing"
"time"
+ "github.com/LatticeNet/lattice-node-agent/internal/linechain"
+ "github.com/LatticeNet/lattice-node-agent/internal/taskexec"
"github.com/LatticeNet/lattice-node-agent/internal/taskoutbox"
"github.com/LatticeNet/lattice-sdk/model"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
+func TestMain(m *testing.M) {
+ if taskexec.MaybeRunChildShim(os.Args) {
+ os.Exit(0)
+ }
+ os.Exit(m.Run())
+}
+
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
@@ -38,8 +48,8 @@ func (r *countingTaskRunner) Run(task model.Task) model.TaskResult {
}
func TestVersionMatchesCurrentRelease(t *testing.T) {
- if version != "0.3.3" {
- t.Fatalf("version = %q, want 0.3.3", version)
+ if version != "0.3.4-alpha.1" {
+ t.Fatalf("version = %q, want 0.3.4-alpha.1", version)
}
}
@@ -48,13 +58,13 @@ func TestCompatibilityPayloadIsEmbedded(t *testing.T) {
if got.ServerMin == "" || got.DashboardMin == "" || got.Channel == "" {
t.Fatalf("compatibility metadata must be embedded: %+v", got)
}
- if got.Channel != "stable" {
- t.Fatalf("compatibility channel = %q, want stable", got.Channel)
+ if got.Channel != "alpha" {
+ t.Fatalf("compatibility channel = %q, want alpha", got.Channel)
}
// The floors stay on design-15 prerelease coordinates on purpose: no stable
// server or dashboard satisfies this agent yet (stable server is still v0.2.1),
// so naming a stable floor here would be a claim the ecosystem cannot back.
- if got.ServerMin != "v0.2.2-alpha.2" || got.DashboardMin != "v0.2.2-alpha.7" {
+ if got.ServerMin != "v0.2.2-alpha.19" || got.DashboardMin != "v0.2.2-alpha.7" {
t.Fatalf("compatibility floor = %+v, want coordinated design-15 alpha", got)
}
}
@@ -258,12 +268,12 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) {
httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
switch r.URL.Path {
case "/api/agent/tasks":
- if r.Header.Get(agentCapabilitiesHeader) != guardManagedSHACapability {
+ if r.Header.Get(agentCapabilitiesHeader) != strings.Join(reportedCapabilities(), ",") {
return testResponse(http.StatusBadRequest, "missing lease-time capability"), nil
}
fetchCalls++
if fetchCalls == 1 {
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
return testResponse(http.StatusOK, string(data)), nil
}
return testResponse(http.StatusOK, `[]`), nil
@@ -284,7 +294,7 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) {
return testResponse(http.StatusNotFound, ""), nil
}
})}
- cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"}
+ cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret", LinechainReady: true}
if err := runTasks(cfg, runner, store); err == nil {
t.Fatal("first result upload should fail")
@@ -308,6 +318,455 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) {
}
}
+type linechainTaskRunner struct {
+ manager *linechain.Manager
+ doc []byte
+ linechainTaskID string
+ calls int
+ callsByID map[string]int
+ afterApply func()
+}
+
+func (r *linechainTaskRunner) Run(task model.Task) model.TaskResult {
+ r.calls++
+ if r.callsByID == nil {
+ r.callsByID = map[string]int{}
+ }
+ r.callsByID[task.ID]++
+ result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, StartedAt: time.Now().UTC(), FinishedAt: time.Now().UTC()}
+ if task.ID != r.linechainTaskID {
+ return result
+ }
+ err := r.manager.Apply(context.Background(), bytes.NewReader(r.doc), task.ID, task.LeaseID, linechainTaskScriptSHA(task.Script))
+ if err == nil && r.afterApply != nil {
+ r.afterApply()
+ }
+ if err != nil {
+ result.ExitCode = -1
+ result.Error = err.Error()
+ }
+ return result
+}
+
+func (r *linechainTaskRunner) RunLinechain(task model.Task) model.TaskResult { return r.Run(task) }
+
+func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) {
+ oldClient := httpClient
+ t.Cleanup(func() { httpClient = oldClient })
+ root := t.TempDir()
+ txnDir := filepath.Join(root, "txn")
+ manager, err := linechain.Open(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer manager.Close()
+ outbox, err := taskoutbox.Open(filepath.Join(root, "outbox"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer outbox.Close()
+ conf := filepath.Join(root, "conf")
+ if err := os.Mkdir(conf, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureLayout(conf, filepath.Join(root, "lattice-metadata.json")); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ writeLinechainSourceSidecar(t, filepath.Join(root, "lattice-metadata.json"))
+ fragmentPath := filepath.Join(conf, "lattice-linechain-0123456789abcdef0123.json")
+ fragment := "{\"outbounds\":[]}"
+ docValue := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, SidecarPatch: testLinechainPatch("create")})
+ doc, err := json.Marshal(map[string]any{
+ "version": docValue.Version, "durable_protocol": docValue.DurableProtocol, "operation": docValue.Operation, "fragment_basename": docValue.FragmentBasename,
+ "fragment": docValue.Fragment, "sidecar_patch": docValue.SidecarPatch, "previous_fragment_sha256": docValue.PreviousFragmentSHA256,
+ "fragment_sha256": docValue.FragmentSHA256, "sidecar_patch_sha256": docValue.SidecarPatchSHA256, "artifact_sha256": docValue.ArtifactSHA256,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: testLinechainApplyScript(doc), TimeoutSec: 10, OutputLimit: 1024}
+ genericTask := model.Task{ID: "task-generic", LeaseID: "lease-generic", Interpreter: "sh", Script: "echo generic"}
+ netguardTask := model.Task{ID: "task-netguard", LeaseID: "lease-netguard", Interpreter: "sh", Script: "echo netguard"}
+ runner := &linechainTaskRunner{manager: manager, doc: doc, linechainTaskID: task.ID, afterApply: func() {
+ manager.ConfigureCleanupForTest(func(path string) error {
+ if strings.HasSuffix(path, ".json") {
+ return errors.New("injected cleanup failure")
+ }
+ return os.Remove(path)
+ }, nil)
+ }}
+ fetches := 0
+ posts := 0
+ var posted []model.TaskResult
+ httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+ switch r.URL.Path {
+ case "/api/agent/tasks":
+ fetches++
+ if fetches == 1 {
+ b, _ := json.Marshal([]leasedAgentTask{
+ {Task: genericTask},
+ {Task: netguardTask, DurableResult: true, DurableProtocol: "netguard-v1"},
+ {Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v2"},
+ })
+ return testResponse(http.StatusOK, string(b)), nil
+ }
+ return testResponse(http.StatusOK, "[]"), nil
+ case "/api/agent/task-result":
+ posts++
+ var body struct {
+ Result model.TaskResult `json:"result"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Fatal(err)
+ }
+ posted = append(posted, body.Result)
+ return testResponse(http.StatusOK, `{"ok":true}`), nil
+ default:
+ return testResponse(http.StatusNotFound, ""), nil
+ }
+ })}
+ cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret", LinechainReady: true}
+ if err := runTasks(cfg, runner, outbox, manager); err == nil {
+ t.Fatal("cleanup failure should remain readiness-visible")
+ }
+ if pending, err := outbox.Pending(); err != nil || len(pending) != 1 {
+ t.Fatalf("completed result not retained: %+v %v", pending, err)
+ }
+ manager.ConfigureCleanupForTest(nil, nil)
+ if err := runTasks(cfg, runner, outbox, manager); err != nil {
+ t.Fatal(err)
+ }
+ if runner.calls != 3 || runner.callsByID[genericTask.ID] != 1 || runner.callsByID[netguardTask.ID] != 1 || runner.callsByID[task.ID] != 1 {
+ t.Fatalf("mixed task execution was replayed or skipped: total=%d byID=%v", runner.calls, runner.callsByID)
+ }
+ if posts != 4 || len(posted) != 4 || posted[0].TaskID != genericTask.ID || posted[1].TaskID != netguardTask.ID || !reflect.DeepEqual(posted[2], posted[3]) {
+ t.Fatalf("mixed result posts=%d results=%+v", posts, posted)
+ }
+ if entries, err := os.ReadDir(txnDir); err != nil {
+ t.Fatal(err)
+ } else {
+ for _, e := range entries {
+ if strings.HasSuffix(e.Name(), ".json") {
+ t.Fatalf("journal not cleaned: %s", e.Name())
+ }
+ }
+ }
+}
+
+func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) {
+ newPair := func(t *testing.T) (*linechain.Manager, *taskoutbox.Store) {
+ t.Helper()
+ root := t.TempDir()
+ manager, err := linechain.Open(filepath.Join(root, "txn"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = manager.Close() })
+ outbox, err := taskoutbox.Open(filepath.Join(root, "outbox"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = outbox.Close() })
+ return manager, outbox
+ }
+ fragment := `{}`
+ doc := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")})
+ rawDoc, err := json.Marshal(map[string]any{
+ "version": doc.Version, "durable_protocol": doc.DurableProtocol, "operation": doc.Operation, "fragment_basename": doc.FragmentBasename,
+ "fragment": doc.Fragment, "sidecar_patch": doc.SidecarPatch, "previous_fragment_sha256": doc.PreviousFragmentSHA256,
+ "fragment_sha256": doc.FragmentSHA256, "sidecar_patch_sha256": doc.SidecarPatchSHA256, "artifact_sha256": doc.ArtifactSHA256,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: testLinechainApplyScript(rawDoc)}
+
+ t.Run("leased E3 without journal blocks require gate", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ if err := requireLinechainRecovered(context.Background(), manager, outbox, "node-a"); err == nil || !strings.Contains(err.Error(), "lacks exact linechain journal") {
+ t.Fatalf("require gate error = %v", err)
+ }
+ })
+
+ t.Run("completed E3 without journal is allowed", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := outbox.Complete(model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}); err != nil {
+ t.Fatal(err)
+ }
+ if err := crossCheckLinechainAuthority(manager, outbox); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ t.Run("journal without exact outbox is blocked", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ createLinechainJournal(t, manager, task)
+ if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "lacks exact") {
+ t.Fatalf("cross-check error = %v", err)
+ }
+ })
+
+ t.Run("journal with wrong protocol is blocked", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "netguard-v1"); err != nil {
+ t.Fatal(err)
+ }
+ createLinechainJournal(t, manager, task)
+ if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "mismatched outbox protocol") {
+ t.Fatalf("cross-check error = %v", err)
+ }
+ })
+
+ t.Run("same IDs with different script are blocked", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ other := task
+ other.Script = "different approved artifact"
+ createLinechainJournal(t, manager, other)
+ if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "exact outbox task script") {
+ t.Fatalf("error=%v", err)
+ }
+ })
+
+ t.Run("journal artifact must match exact issued script", func(t *testing.T) {
+ root := t.TempDir()
+ txnDir := filepath.Join(root, "txn")
+ manager, err := linechain.Open(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = manager.Close() })
+ outbox, err := taskoutbox.Open(filepath.Join(root, "outbox"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = outbox.Close() })
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ createLinechainJournal(t, manager, task)
+ entries, err := os.ReadDir(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, entry := range entries {
+ if !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ path := filepath.Join(txnDir, entry.Name())
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var journal map[string]any
+ if err := json.Unmarshal(raw, &journal); err != nil {
+ t.Fatal(err)
+ }
+ journal["artifact_sha256"] = strings.Repeat("a", 64)
+ tampered, err := json.Marshal(journal)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, tampered, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "issued artifact") {
+ t.Fatalf("tampered journal artifact error = %v", err)
+ }
+ })
+
+ t.Run("completed E3 requires exact terminal result", func(t *testing.T) {
+ for _, mismatch := range []bool{false, true} {
+ t.Run(map[bool]string{false: "matching", true: "mismatched"}[mismatch], func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ createLinechainJournal(t, manager, task)
+ result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}
+ if _, err := manager.ResolveAfterRun(context.Background(), task, result); err != nil {
+ t.Fatal(err)
+ }
+ outboxResult := result
+ if mismatch {
+ outboxResult.ExitCode = -1
+ outboxResult.Error = "different"
+ }
+ if _, err := outbox.Complete(outboxResult); err != nil {
+ t.Fatal(err)
+ }
+ err := crossCheckLinechainAuthority(manager, outbox)
+ if mismatch && (err == nil || !strings.Contains(err.Error(), "conflicts")) {
+ t.Fatalf("error=%v", err)
+ }
+ if !mismatch && err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ })
+
+ t.Run("completed E3 conflicts with nonterminal journal", func(t *testing.T) {
+ manager, outbox := newPair(t)
+ if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil {
+ t.Fatal(err)
+ }
+ createLinechainJournal(t, manager, task)
+ if _, err := outbox.Complete(model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}); err != nil {
+ t.Fatal(err)
+ }
+ if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "conflicts") {
+ t.Fatalf("error=%v", err)
+ }
+ })
+}
+
+func createLinechainJournal(t *testing.T, manager *linechain.Manager, task model.Task) {
+ t.Helper()
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ if err := os.Mkdir(configDir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ writeLinechainSourceSidecar(t, filepath.Join(root, "lattice-metadata.json"))
+ fragment := `{}`
+ d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")})
+ raw, err := json.Marshal(map[string]any{
+ "version": d.Version, "durable_protocol": d.DurableProtocol, "operation": d.Operation, "fragment_basename": d.FragmentBasename,
+ "fragment": d.Fragment, "sidecar_patch": d.SidecarPatch, "previous_fragment_sha256": d.PreviousFragmentSHA256,
+ "fragment_sha256": d.FragmentSHA256, "sidecar_patch_sha256": d.SidecarPatchSHA256, "artifact_sha256": d.ArtifactSHA256,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.Apply(context.Background(), bytes.NewReader(raw), task.ID, task.LeaseID, linechainTaskScriptSHA(task.Script)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestTaskexecRunsRealLinechainHelperShell(t *testing.T) {
+ root := t.TempDir()
+ txnDir := filepath.Join(root, "txn")
+ configDir := filepath.Join(root, "conf")
+ sidecarPath := filepath.Join(root, "lattice-metadata.json")
+ workdir := filepath.Join(root, "work")
+ for _, dir := range []string{configDir, workdir} {
+ if err := os.Mkdir(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ }
+ manager, err := linechain.Open(txnDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer manager.Close()
+ if err := manager.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ writeLinechainSourceSidecar(t, sidecarPath)
+ fragment := `{"outbounds":[]}`
+ d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")})
+ raw, err := json.Marshal(map[string]any{
+ "version": d.Version, "durable_protocol": d.DurableProtocol, "operation": d.Operation, "fragment_basename": d.FragmentBasename,
+ "fragment": d.Fragment, "sidecar_patch": d.SidecarPatch, "previous_fragment_sha256": d.PreviousFragmentSHA256,
+ "fragment_sha256": d.FragmentSHA256, "sidecar_patch_sha256": d.SidecarPatchSHA256, "artifact_sha256": d.ArtifactSHA256,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ testBinary, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ task := model.Task{
+ ID: "task-real-shell", LeaseID: "lease-real-shell", Interpreter: "sh",
+ Script: "printf '%s' '" + string(raw) + "' | LATTICE_TEST_LINECHAIN_HELPER=1 \"$LATTICE_AGENT_BIN\" -test.run=TestTaskexecLinechainHelperProcess",
+ TimeoutSec: 20, OutputLimit: 4096,
+ }
+ runner := taskexec.Runner{
+ AllowExec: true, AllowRoot: true, WorkdirRoot: workdir, AgentBinary: testBinary,
+ LinechainTxnDir: txnDir, LinechainConfigDir: configDir, LinechainSidecarPath: sidecarPath,
+ }
+ result := runner.RunLinechain(task)
+ if result.ExitCode != 0 || result.Error != "" {
+ t.Fatalf("real taskexec helper failed: %+v stderr=%s", result, result.Stderr)
+ }
+ result, err = manager.ResolveAfterRun(context.Background(), task, result)
+ if err != nil || result.ExitCode != 0 {
+ t.Fatalf("resolve real taskexec helper: result=%+v err=%v", result, err)
+ }
+ if err := manager.Cleanup(task.ID, task.LeaseID); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestTaskexecLinechainHelperProcess(t *testing.T) {
+ if os.Getenv("LATTICE_TEST_LINECHAIN_HELPER") != "1" {
+ return
+ }
+ manager, err := linechain.OpenHelper(os.Getenv("LATTICE_LINECHAIN_TXN_DIR"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureLayout(os.Getenv("LATTICE_LINECHAIN_CONFIG_DIR"), os.Getenv("LATTICE_LINECHAIN_SIDECAR_PATH")); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID"), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func writeLinechainSourceSidecar(t *testing.T, path string) {
+ t.Helper()
+ const sidecar = `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"11111111-1111-4111-8111-1111111111aa"}]}`
+ if err := os.WriteFile(path, []byte(sidecar), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testLinechainPatch(operation string) linechain.SidecarPatchV2 {
+ desired := "33333333-3333-4333-8333-333333333333"
+ patch := linechain.SidecarPatchV2{
+ Schema: "lattice.singbox-linechain-sidecar-patch.v1", SourceLineUUID: "11111111-1111-4111-8111-1111111111aa",
+ SourceInboundTag: "source", DesiredDownstreamLineUUID: &desired,
+ }
+ if operation != "create" {
+ patch.ExpectedDownstreamLineUUID = &desired
+ }
+ if operation == "remove" {
+ patch.DesiredDownstreamLineUUID = nil
+ }
+ return patch
+}
+
+func testLinechainApplyScript(document []byte) string {
+ return "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '" + base64.StdEncoding.EncodeToString(document) + "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n"
+}
+
func TestRunTasksRestartFlushesCompletedResultBeforeFetch(t *testing.T) {
oldClient := httpClient
defer func() { httpClient = oldClient }()
@@ -416,7 +875,7 @@ func TestRunTasksJournalFailurePreventsExecution(t *testing.T) {
oldClient := httpClient
defer func() { httpClient = oldClient }()
task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must not run"}
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
var reported model.TaskResult
httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.URL.Path == "/api/agent/tasks" {
@@ -453,7 +912,7 @@ func TestRunTasksPublishedJournalFailureDoesNotPostConflictingDirectResult(t *te
oldClient := httpClient
defer func() { httpClient = oldClient }()
task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must not run"}
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
postCalls := 0
httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.URL.Path == "/api/agent/tasks" {
@@ -480,7 +939,7 @@ func TestRunTasksExactRedeliveryDoesNotExecuteExistingJournal(t *testing.T) {
oldClient := httpClient
defer func() { httpClient = oldClient }()
task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must run once"}
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.URL.Path != "/api/agent/tasks" {
t.Fatalf("unexpected request for already-journaled lease: %s", r.URL.Path)
@@ -530,6 +989,24 @@ func TestRunTasksKeepsGenericTasksOutsideDurableNetGuardProtocol(t *testing.T) {
}
}
+func TestRunTasksRejectsUnknownDurableProtocolBeforeExecution(t *testing.T) {
+ oldClient := httpClient
+ defer func() { httpClient = oldClient }()
+ task := model.Task{ID: "task-unknown", LeaseID: "lease-unknown", Interpreter: "sh", Script: "must not run"}
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "unknown-v1"}})
+ httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+ return testResponse(http.StatusOK, string(data)), nil
+ })}
+ runner := &countingTaskRunner{}
+ err := runTasks(agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"}, runner, beginFailingOutbox{})
+ if err == nil || !strings.Contains(err.Error(), "unsupported protocol") {
+ t.Fatalf("error=%v", err)
+ }
+ if runner.calls != 0 {
+ t.Fatalf("unknown protocol executed: %d", runner.calls)
+ }
+}
+
type completePublishingOutbox struct {
task model.Task
result *model.TaskResult
@@ -572,7 +1049,7 @@ func TestRunTasksConfirmsAndUploadsResultPublishedBeforeDirectorySyncFailure(t *
oldClient := httpClient
defer func() { httpClient = oldClient }()
task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo once"}
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
posts := 0
var posted model.TaskResult
outbox := &completePublishingOutbox{}
@@ -614,7 +1091,7 @@ func TestRunTasksDoesNotUploadUnconfirmedPublishedResult(t *testing.T) {
oldClient := httpClient
defer func() { httpClient = oldClient }()
task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo once"}
- data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}})
+ data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}})
posts := 0
httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
switch r.URL.Path {
diff --git a/go.mod b/go.mod
index 25485ed..b9fa357 100644
--- a/go.mod
+++ b/go.mod
@@ -8,13 +8,13 @@ require github.com/creack/pty v1.1.24
require (
github.com/gorilla/websocket v1.5.3
+ golang.org/x/sys v0.43.0
google.golang.org/grpc v1.82.1
google.golang.org/protobuf v1.36.11
)
require (
golang.org/x/net v0.53.0 // indirect
- golang.org/x/sys v0.43.0 // indirect
golang.org/x/text v0.36.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
)
diff --git a/internal/linechain/lock_other.go b/internal/linechain/lock_other.go
new file mode 100644
index 0000000..9455bbd
--- /dev/null
+++ b/internal/linechain/lock_other.go
@@ -0,0 +1,18 @@
+//go:build !unix
+
+package linechain
+
+import "os"
+
+func lockManager(path string) (*os.File, error) {
+ return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600)
+}
+func unlockManager(f *os.File) error {
+ name := f.Name()
+ err := f.Close()
+ if removeErr := os.Remove(name); err == nil {
+ err = removeErr
+ }
+ return err
+}
+func ownedPath(os.FileInfo) bool { return true }
diff --git a/internal/linechain/lock_unix.go b/internal/linechain/lock_unix.go
new file mode 100644
index 0000000..cf63d4c
--- /dev/null
+++ b/internal/linechain/lock_unix.go
@@ -0,0 +1,65 @@
+//go:build unix
+
+package linechain
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+)
+
+func lockManager(path string) (*os.File, error) {
+ fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600)
+ if err != nil {
+ return nil, err
+ }
+ f := os.NewFile(uintptr(fd), path)
+ if f == nil {
+ _ = syscall.Close(fd)
+ return nil, fmt.Errorf("open linechain manager lock: invalid file descriptor")
+ }
+ info, err := f.Stat()
+ if err != nil {
+ _ = f.Close()
+ return nil, fmt.Errorf("inspect linechain manager lock: %w", err)
+ }
+ if err := validateLockInfo(info, os.Geteuid()); err != nil {
+ _ = f.Close()
+ return nil, err
+ }
+ if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+ _ = f.Close()
+ return nil, fmt.Errorf("linechain transaction manager already open: %w", err)
+ }
+ return f, nil
+}
+
+func validateLockInfo(info os.FileInfo, effectiveUID int) error {
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return fmt.Errorf("inspect linechain manager lock ownership: unsupported stat data")
+ }
+ if !info.Mode().IsRegular() {
+ return fmt.Errorf("linechain manager lock must be a regular file")
+ }
+ if stat.Uid != uint32(effectiveUID) {
+ return fmt.Errorf("linechain manager lock must be owned by effective user %d", effectiveUID)
+ }
+ if info.Mode().Perm() != 0o600 {
+ return fmt.Errorf("linechain manager lock permissions are %o, want 600", info.Mode().Perm())
+ }
+ return nil
+}
+
+func unlockManager(f *os.File) error {
+ err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+ if closeErr := f.Close(); err == nil {
+ err = closeErr
+ }
+ return err
+}
+
+func ownedPath(info os.FileInfo) bool {
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ return ok && int(stat.Uid) == os.Geteuid()
+}
diff --git a/internal/linechain/lock_unix_test.go b/internal/linechain/lock_unix_test.go
new file mode 100644
index 0000000..7909fb6
--- /dev/null
+++ b/internal/linechain/lock_unix_test.go
@@ -0,0 +1,83 @@
+//go:build linux || darwin || freebsd
+
+package linechain
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+func TestOpenRefusesSymlinkLockWithoutChangingTarget(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "txn")
+ if err := os.Mkdir(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ victim := filepath.Join(t.TempDir(), "victim")
+ if err := os.WriteFile(victim, []byte("unchanged"), 0o640); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(victim, filepath.Join(dir, ".lock")); err != nil {
+ t.Fatal(err)
+ }
+ if m, err := Open(dir); err == nil {
+ _ = m.Close()
+ t.Fatal("symlink lock accepted")
+ }
+ b, err := os.ReadFile(victim)
+ if err != nil || string(b) != "unchanged" {
+ t.Fatalf("victim changed: %q %v", b, err)
+ }
+}
+
+func TestOpenRefusesFIFOAndInsecureLockMode(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ make func(string) error
+ }{
+ {"fifo", func(path string) error { return syscall.Mkfifo(path, 0o600) }},
+ {"mode", func(path string) error { return os.WriteFile(path, nil, 0o644) }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "txn")
+ if err := os.Mkdir(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ path := filepath.Join(dir, ".lock")
+ if err := tc.make(path); err != nil {
+ t.Fatal(err)
+ }
+ if tc.name == "mode" {
+ _ = os.Chmod(path, 0o644)
+ }
+ if m, err := Open(dir); err == nil {
+ _ = m.Close()
+ t.Fatal("hostile lock accepted")
+ }
+ })
+ }
+}
+
+type lockInfoWithStat struct {
+ os.FileInfo
+ stat syscall.Stat_t
+}
+
+func (i lockInfoWithStat) Sys() any { return &i.stat }
+
+func TestLockValidationRefusesWrongOwner(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "lock")
+ if err := os.WriteFile(path, nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stat := *(info.Sys().(*syscall.Stat_t))
+ stat.Uid = uint32(os.Geteuid() + 1)
+ if err := validateLockInfo(lockInfoWithStat{FileInfo: info, stat: stat}, os.Geteuid()); err == nil {
+ t.Fatal("wrong-owner lock accepted")
+ }
+}
diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go
new file mode 100644
index 0000000..42270b8
--- /dev/null
+++ b/internal/linechain/manager.go
@@ -0,0 +1,1075 @@
+// Package linechain applies the two host-local artifacts that define a
+// sing-box line chain as one crash-recoverable transaction.
+package linechain
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/LatticeNet/lattice-sdk/model"
+)
+
+const (
+ journalVersion = 2
+ maxDocumentSize = 4 << 20
+ maxArtifactSize = maxDocumentSize
+)
+
+var linechainBasenameRE = regexp.MustCompile(`^lattice-linechain-[0-9a-f]{20}\.json$`)
+
+// Document is the bounded server-rendered input consumed by -linechain-apply.
+// Desired content is never copied into the journal; only its digest is stored.
+type Document struct {
+ Version int `json:"version"`
+ DurableProtocol string `json:"durable_protocol"`
+ Operation string `json:"operation"`
+ ConfigDir string `json:"-"`
+ FragmentBasename string `json:"fragment_basename"`
+ FragmentPath string `json:"-"`
+ SidecarPath string `json:"-"`
+ Fragment *string `json:"fragment"`
+ SidecarPatch SidecarPatchV2 `json:"sidecar_patch"`
+ PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"`
+ FragmentSHA256 *string `json:"fragment_sha256"`
+ SidecarPatchSHA256 string `json:"sidecar_patch_sha256"`
+ ArtifactSHA256 string `json:"artifact_sha256"`
+ SidecarPatchCanonical []byte `json:"-"`
+ SidecarOutput *string `json:"-"`
+}
+
+// wireDocumentV2 is the only server-controlled shape accepted by Apply.
+// Host paths and the ordinary-writer sidecar predecessor are intentionally not
+// representable on the wire; Apply derives paths from the local layout.
+type wireDocumentV2 struct {
+ Version int `json:"version"`
+ DurableProtocol string `json:"durable_protocol"`
+ Operation string `json:"operation"`
+ FragmentBasename string `json:"fragment_basename"`
+ Fragment *string `json:"fragment"`
+ SidecarPatch SidecarPatchV2 `json:"sidecar_patch"`
+ PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"`
+ FragmentSHA256 *string `json:"fragment_sha256"`
+ SidecarPatchSHA256 string `json:"sidecar_patch_sha256"`
+ ArtifactSHA256 string `json:"artifact_sha256"`
+}
+
+// BindDocument computes canonical test/server fixture bindings. Apply validates
+// issued bindings and never calls this after reading or merging host state.
+func BindDocument(d Document) Document {
+ if d.DurableProtocol == "" {
+ d.DurableProtocol = "linechain-e3-v2"
+ }
+ if d.Fragment == nil {
+ d.FragmentSHA256 = nil
+ } else {
+ value := digest([]byte(*d.Fragment))
+ d.FragmentSHA256 = &value
+ }
+ patch, _ := json.Marshal(d.SidecarPatch)
+ d.SidecarPatchCanonical = patch
+ d.SidecarPatchSHA256 = digest(patch)
+ binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: d.Operation, FragmentBasename: d.FragmentBasename,
+ PreviousFragmentSHA256: d.PreviousFragmentSHA256, FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256}
+ canonical, _ := canonicalSemanticArtifactBinding(binding)
+ d.ArtifactSHA256 = digest(canonical)
+ return d
+}
+
+type journal struct {
+ Version int `json:"version"`
+ TaskID string `json:"task_id"`
+ LeaseID string `json:"lease_id"`
+ FragmentPath string `json:"fragment_path"`
+ SidecarPath string `json:"sidecar_path"`
+ FragmentOld string `json:"fragment_old_sha256,omitempty"`
+ SidecarOld string `json:"sidecar_old_sha256,omitempty"`
+ ArtifactSHA256 string `json:"artifact_sha256"`
+ SidecarPatchSHA256 string `json:"sidecar_patch_sha256"`
+ FragmentOutputSHA256 string `json:"fragment_output_sha256,omitempty"`
+ SidecarOutputSHA256 string `json:"sidecar_output_sha256"`
+ TaskScriptSHA string `json:"task_script_sha256"`
+ FragmentHadOld bool `json:"fragment_had_old"`
+ SidecarHadOld bool `json:"sidecar_had_old"`
+ Phase string `json:"phase"`
+ Result *model.TaskResult `json:"result,omitempty"`
+}
+
+type Manager struct {
+ dir string
+ lock *os.File
+ run func(context.Context, string, ...string) ([]byte, error)
+ configDir string
+ sidecarPath string
+ remove func(string) error
+ publishFile func(string, *string) error
+ writeJournal func(string, any) error
+ syncDirectory func(string) error
+ singBoxBinary string
+ restartCommand []string
+ verifyCommand []string
+}
+
+func (m *Manager) ConfigureLayout(configDir, sidecarPath string) error {
+ configDir = filepath.Clean(strings.TrimSpace(configDir))
+ sidecarPath = filepath.Clean(strings.TrimSpace(sidecarPath))
+ if !filepath.IsAbs(configDir) || !filepath.IsAbs(sidecarPath) {
+ return fmt.Errorf("linechain layout paths must be absolute")
+ }
+ resolvedConfig, err := filepath.EvalSymlinks(configDir)
+ if err != nil {
+ return fmt.Errorf("resolve linechain config directory: %w", err)
+ }
+ resolvedSidecarParent, err := filepath.EvalSymlinks(filepath.Dir(sidecarPath))
+ if err != nil {
+ return fmt.Errorf("resolve linechain sidecar directory: %w", err)
+ }
+ for label, root := range map[string]string{"config": resolvedConfig, "sidecar": resolvedSidecarParent} {
+ info, err := os.Lstat(root)
+ if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) || info.Mode().Perm()&0o022 != 0 {
+ return fmt.Errorf("linechain %s root is not a trusted private directory", label)
+ }
+ }
+ configDir = filepath.Clean(resolvedConfig)
+ sidecarPath = filepath.Join(filepath.Clean(resolvedSidecarParent), filepath.Base(sidecarPath))
+ if err := validateParents(filepath.Join(configDir, "placeholder")); err != nil {
+ return err
+ }
+ if err := validateParents(sidecarPath); err != nil {
+ return err
+ }
+ m.configDir = configDir
+ m.sidecarPath = sidecarPath
+ return nil
+}
+func (m *Manager) Configured() bool { return m != nil && m.configDir != "" && m.sidecarPath != "" }
+
+// ConfigureCommands is an integration-test seam. Production uses fixed local
+// command vectors; task documents cannot select executables or arguments.
+func (m *Manager) ConfigureCommands(binary string, restart, verify []string) error {
+ if strings.TrimSpace(binary) == "" || len(restart) == 0 || len(verify) == 0 {
+ return fmt.Errorf("linechain command vectors must be non-empty")
+ }
+ m.singBoxBinary = binary
+ m.restartCommand = append([]string(nil), restart...)
+ m.verifyCommand = append([]string(nil), verify...)
+ return nil
+}
+
+// ConfigureCleanupForTest injects cleanup fault seams for crash-boundary tests.
+func (m *Manager) ConfigureCleanupForTest(remove func(string) error, syncDirectory func(string) error) {
+ if remove == nil {
+ remove = os.Remove
+ }
+ if syncDirectory == nil {
+ syncDirectory = syncDir
+ }
+ m.remove = remove
+ m.syncDirectory = syncDirectory
+}
+
+// ConfigureMutationForTest injects atomic publication and journal-write faults.
+func (m *Manager) ConfigureMutationForTest(publishFile func(string, *string) error, writeJournal func(string, any) error) {
+ if publishFile == nil {
+ publishFile = publish
+ }
+ if writeJournal == nil {
+ writeJournal = writeJSON
+ }
+ m.publishFile = publishFile
+ m.writeJournal = writeJournal
+}
+
+// Open takes exclusive ownership of a private absolute transaction directory.
+func Open(dir string) (*Manager, error) {
+ m, err := open(dir)
+ if err != nil {
+ return nil, err
+ }
+ lock, err := lockManager(filepath.Join(m.dir, ".lock"))
+ if err != nil {
+ return nil, fmt.Errorf("linechain transaction manager already open: %w", err)
+ }
+ m.lock = lock
+ if err := syncDir(m.dir); err != nil {
+ _ = unlockManager(lock)
+ m.lock = nil
+ return nil, err
+ }
+ return m, nil
+}
+
+// OpenHelper opens the transaction directory without taking the manager lock.
+// It is used only by a child helper spawned by the lock-owning agent.
+func OpenHelper(dir string) (*Manager, error) { return open(dir) }
+
+func open(dir string) (*Manager, error) {
+ dir = filepath.Clean(strings.TrimSpace(dir))
+ if dir == "." || !filepath.IsAbs(dir) || dir == string(filepath.Separator) {
+ return nil, fmt.Errorf("linechain transaction directory must be absolute and non-root")
+ }
+ if err := makeDurablePrivateDir(dir, syncDir); err != nil {
+ return nil, err
+ }
+ info, err := os.Lstat(dir)
+ if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
+ return nil, fmt.Errorf("linechain transaction path must be a real directory")
+ }
+ if !ownedPath(info) || info.Mode().Perm() != 0o700 {
+ return nil, fmt.Errorf("linechain transaction directory must be agent-owned with exact mode 0700")
+ }
+ m := &Manager{dir: dir}
+ m.remove = os.Remove
+ m.publishFile = publish
+ m.writeJournal = writeJSON
+ m.syncDirectory = syncDir
+ m.singBoxBinary = "sing-box"
+ m.restartCommand = []string{"systemctl", "restart", "sing-box"}
+ m.verifyCommand = []string{"systemctl", "is-active", "--quiet", "sing-box"}
+ m.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
+ return exec.CommandContext(ctx, name, args...).CombinedOutput()
+ }
+ return m, nil
+}
+
+func makeDurablePrivateDir(dir string, syncDirectory func(string) error) error {
+ dir = filepath.Clean(dir)
+ missing := []string{}
+ current := dir
+ for {
+ info, err := os.Lstat(current)
+ if err == nil {
+ if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+ return fmt.Errorf("path component must be a real directory: %s", current)
+ }
+ break
+ }
+ if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ missing = append(missing, current)
+ parent := filepath.Dir(current)
+ if parent == current {
+ return fmt.Errorf("no existing parent for linechain transaction directory")
+ }
+ current = parent
+ }
+ if parent := filepath.Dir(current); parent != current {
+ if err := syncDirectory(parent); err != nil {
+ return fmt.Errorf("confirm existing directory %s: %w", current, err)
+ }
+ }
+ for i := len(missing) - 1; i >= 0; i-- {
+ path := missing[i]
+ if err := os.Mkdir(path, 0o700); err != nil {
+ return err
+ }
+ if err := syncDirectory(filepath.Dir(path)); err != nil {
+ return fmt.Errorf("sync parent after creating %s: %w", path, err)
+ }
+ }
+ return nil
+}
+
+func (m *Manager) Close() error {
+ if m == nil || m.lock == nil {
+ return nil
+ }
+ err := unlockManager(m.lock)
+ m.lock = nil
+ return err
+}
+
+func (m *Manager) journalPath(taskID, leaseID string) string {
+ s := sha256.Sum256([]byte(taskID + "\x00" + leaseID))
+ return filepath.Join(m.dir, hex.EncodeToString(s[:])+".json")
+}
+
+// Apply reads and applies one document. It is intended for the early-exit
+// helper mode and obtains task identity from the runner's minimal environment.
+func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskScriptSHA string) error {
+ if strings.TrimSpace(taskID) == "" || strings.TrimSpace(leaseID) == "" {
+ return fmt.Errorf("task and lease identity are required")
+ }
+ scriptBinding := strings.TrimSpace(taskScriptSHA)
+ if !validSHA(scriptBinding) {
+ return fmt.Errorf("linechain task script binding is required and must be lowercase SHA-256")
+ }
+ raw, err := io.ReadAll(io.LimitReader(r, maxDocumentSize+1))
+ if err != nil {
+ return fmt.Errorf("read linechain document: %w", err)
+ }
+ if len(raw) > maxDocumentSize {
+ return fmt.Errorf("linechain document exceeds %d bytes", maxDocumentSize)
+ }
+ dec := json.NewDecoder(bytes.NewReader(raw))
+ dec.DisallowUnknownFields()
+ var wire wireDocumentV2
+ if err := dec.Decode(&wire); err != nil {
+ return fmt.Errorf("decode linechain document: %w", err)
+ }
+ var trailing any
+ if err := dec.Decode(&trailing); err == nil {
+ return fmt.Errorf("decode linechain document: unexpected trailing data")
+ } else if !errors.Is(err, io.EOF) {
+ return fmt.Errorf("decode linechain document trailing data: %w", err)
+ }
+ if wire.Version != 2 {
+ return fmt.Errorf("unsupported linechain document version %d", wire.Version)
+ }
+ if wire.DurableProtocol != "linechain-e3-v2" {
+ return fmt.Errorf("unsupported linechain durable protocol %q", wire.DurableProtocol)
+ }
+ rawFields, err := decodeUniqueJSONObject(raw)
+ if err != nil {
+ return fmt.Errorf("decode linechain document fields: %w", err)
+ }
+ for _, name := range []string{"version", "durable_protocol", "operation", "fragment_basename", "fragment", "sidecar_patch", "previous_fragment_sha256", "fragment_sha256", "sidecar_patch_sha256", "artifact_sha256"} {
+ if _, ok := rawFields[name]; !ok {
+ return fmt.Errorf("linechain document field %s must be present", name)
+ }
+ }
+ if filepath.Base(wire.FragmentBasename) != wire.FragmentBasename || !linechainBasenameRE.MatchString(wire.FragmentBasename) {
+ return fmt.Errorf("fragment_basename is invalid")
+ }
+ patch, patchCanonical, err := canonicalSemanticSidecarPatch(rawFields["sidecar_patch"])
+ if err != nil {
+ return err
+ }
+ binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: wire.Operation, FragmentBasename: wire.FragmentBasename,
+ PreviousFragmentSHA256: wire.PreviousFragmentSHA256, FragmentSHA256: wire.FragmentSHA256, SidecarPatchSHA256: wire.SidecarPatchSHA256}
+ if err := verifySemanticArtifactBinding(wire.Fragment, patchCanonical, binding, wire.ArtifactSHA256); err != nil {
+ return err
+ }
+ d := Document{
+ Version: wire.Version, DurableProtocol: wire.DurableProtocol, Operation: wire.Operation, ConfigDir: m.configDir,
+ FragmentBasename: wire.FragmentBasename, FragmentPath: filepath.Join(m.configDir, wire.FragmentBasename), SidecarPath: m.sidecarPath,
+ PreviousFragmentSHA256: wire.PreviousFragmentSHA256, Fragment: wire.Fragment, SidecarPatch: patch,
+ FragmentSHA256: wire.FragmentSHA256, SidecarPatchSHA256: wire.SidecarPatchSHA256, ArtifactSHA256: wire.ArtifactSHA256,
+ SidecarPatchCanonical: patchCanonical,
+ }
+ if err := m.validateDocument(d); err != nil {
+ return err
+ }
+ path := m.journalPath(taskID, leaseID)
+ if _, err := os.Lstat(path); err == nil {
+ return m.recoverOne(ctx, path, nil, "", scriptBinding)
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+
+ fragmentOld, fragmentHad, err := readCurrent(d.FragmentPath)
+ if err != nil {
+ return err
+ }
+ sidecarOld, sidecarHad, err := readCurrent(d.SidecarPath)
+ if err != nil {
+ return err
+ }
+ previousFragmentSHA := ""
+ if d.PreviousFragmentSHA256 != nil {
+ previousFragmentSHA = *d.PreviousFragmentSHA256
+ }
+ if err := requirePrevious("fragment", fragmentOld, fragmentHad, previousFragmentSHA); err != nil {
+ return err
+ }
+ if !sidecarHad {
+ return fmt.Errorf("current semantic sidecar is required")
+ }
+ mergedSidecar, err := mergeManagedSidecar(sidecarOld, d.SidecarPatch)
+ if err != nil {
+ return err
+ }
+ mergedSidecarText := string(mergedSidecar)
+ d.SidecarOutput = &mergedSidecarText
+ j := journal{Version: journalVersion, TaskID: taskID, LeaseID: leaseID, FragmentPath: d.FragmentPath, SidecarPath: d.SidecarPath,
+ FragmentOld: digestMaybe(fragmentOld, fragmentHad), SidecarOld: digestMaybe(sidecarOld, sidecarHad), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad,
+ ArtifactSHA256: d.ArtifactSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256,
+ FragmentOutputSHA256: digestPtr(d.Fragment), SidecarOutputSHA256: digestPtr(d.SidecarOutput), TaskScriptSHA: scriptBinding, Phase: "prepared"}
+ if err := m.writeBackup(path+".fragment.old", fragmentOld, fragmentHad); err != nil {
+ return err
+ }
+ if err := m.writeBackup(path+".sidecar.old", sidecarOld, sidecarHad); err != nil {
+ return err
+ }
+ if err := m.writeJournal(path, j); err != nil {
+ return err
+ }
+ if err := m.publishFile(d.FragmentPath, d.Fragment); err != nil {
+ return m.rollback(ctx, path, &j, d, fmt.Errorf("publish fragment: %w", err))
+ }
+ j.Phase = "fragment_published"
+ if err := m.writeJournal(path, j); err != nil {
+ return err
+ }
+ if err := m.publishFile(d.SidecarPath, d.SidecarOutput); err != nil {
+ return m.rollback(ctx, path, &j, d, fmt.Errorf("publish sidecar: %w", err))
+ }
+ j.Phase = "pair_published"
+ if err := m.writeJournal(path, j); err != nil {
+ return err
+ }
+ if err := m.checkRestartVerify(ctx, d); err != nil {
+ return m.rollback(ctx, path, &j, d, err)
+ }
+ j.Phase = "desired_verified"
+ return m.writeJournal(path, j)
+}
+
+func decodeUniqueJSONObject(raw []byte) (map[string]json.RawMessage, error) {
+ dec := json.NewDecoder(bytes.NewReader(raw))
+ token, err := dec.Token()
+ if err != nil || token != json.Delim('{') {
+ return nil, fmt.Errorf("must be an object")
+ }
+ fields := make(map[string]json.RawMessage)
+ for dec.More() {
+ token, err := dec.Token()
+ if err != nil {
+ return nil, err
+ }
+ name, ok := token.(string)
+ if !ok {
+ return nil, fmt.Errorf("object field name is invalid")
+ }
+ if _, exists := fields[name]; exists {
+ return nil, fmt.Errorf("duplicate field %s", name)
+ }
+ var value json.RawMessage
+ if err := dec.Decode(&value); err != nil {
+ return nil, err
+ }
+ fields[name] = value
+ }
+ if _, err := dec.Token(); err != nil {
+ return nil, err
+ }
+ if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) {
+ return nil, fmt.Errorf("unexpected trailing data")
+ }
+ return fields, nil
+}
+
+// JournalRef is the bounded identity needed to cross-check outbox authority.
+type JournalRef struct {
+ TaskID, LeaseID string
+ Phase string
+ Result *model.TaskResult
+ Terminal bool
+ TaskScriptSHA string
+ ArtifactSHA256 string
+ JournalSHA256 string
+}
+
+type RecoveryAuthority struct {
+ TaskScriptSHA, ArtifactSHA256, Phase, ResultSHA256, JournalSHA256 string
+}
+
+func (m *Manager) Snapshot() ([]JournalRef, error) {
+ entries, err := os.ReadDir(m.dir)
+ if err != nil {
+ return nil, err
+ }
+ refs := make([]JournalRef, 0, len(entries))
+ seen := make(map[string]struct{}, len(entries))
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ if len(refs) >= 1024 {
+ return nil, fmt.Errorf("linechain journal capacity exceeded")
+ }
+ j, err := readJournal(filepath.Join(m.dir, entry.Name()))
+ if err != nil {
+ return nil, err
+ }
+ key := j.TaskID + "\x00" + j.LeaseID
+ if _, ok := seen[key]; ok {
+ return nil, fmt.Errorf("duplicate linechain journal identity: %s", j.TaskID)
+ }
+ seen[key] = struct{}{}
+ if entry.Name() != filepath.Base(m.journalPath(j.TaskID, j.LeaseID)) {
+ return nil, fmt.Errorf("linechain journal filename does not match task and lease: %s", entry.Name())
+ }
+ if err := m.validateJournal(filepath.Join(m.dir, entry.Name()), j); err != nil {
+ return nil, err
+ }
+ refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID, Phase: j.Phase, Result: j.Result, Terminal: journalTerminal(j.Phase), TaskScriptSHA: j.TaskScriptSHA, ArtifactSHA256: j.ArtifactSHA256, JournalSHA256: journalSHA(j)})
+ }
+ return refs, nil
+}
+
+func (m *Manager) validateJournal(path string, j journal) error {
+ if !m.Configured() {
+ return fmt.Errorf("linechain journal exists before runtime layout is configured")
+ }
+ if filepath.Clean(path) != m.journalPath(j.TaskID, j.LeaseID) {
+ return fmt.Errorf("linechain journal filename does not match task and lease")
+ }
+ fragmentName := filepath.Base(j.FragmentPath)
+ if filepath.Clean(j.FragmentPath) != filepath.Join(m.configDir, fragmentName) || !linechainBasenameRE.MatchString(fragmentName) {
+ return fmt.Errorf("linechain journal fragment path is outside local authority")
+ }
+ if filepath.Clean(j.SidecarPath) != m.sidecarPath {
+ return fmt.Errorf("linechain journal sidecar path is outside local authority")
+ }
+ validPhase := map[string]bool{
+ "prepared": true, "fragment_published": true, "pair_published": true, "desired_verified": true,
+ "old_restored": true, "terminal_desired": true, "terminal_old": true,
+ }
+ if !validPhase[j.Phase] {
+ return fmt.Errorf("linechain journal has unknown phase %q", j.Phase)
+ }
+ terminal := journalTerminal(j.Phase)
+ if terminal != (j.Result != nil) {
+ return fmt.Errorf("linechain journal phase/result shape is inconsistent")
+ }
+ if j.Result != nil && (j.Result.TaskID != j.TaskID || j.Result.LeaseID != j.LeaseID || j.Result.FinishedAt.IsZero()) {
+ return fmt.Errorf("linechain journal terminal result does not match its identity")
+ }
+ if (j.FragmentHadOld && !validSHA(j.FragmentOld)) || (!j.FragmentHadOld && j.FragmentOld != "") ||
+ (j.SidecarHadOld && !validSHA(j.SidecarOld)) || (!j.SidecarHadOld && j.SidecarOld != "") ||
+ (j.FragmentOutputSHA256 != "" && !validSHA(j.FragmentOutputSHA256)) || !validSHA(j.SidecarOutputSHA256) ||
+ !validSHA(j.SidecarPatchSHA256) || !validSHA(j.ArtifactSHA256) || !validSHA(j.TaskScriptSHA) {
+ return fmt.Errorf("linechain journal artifact digest shape is invalid")
+ }
+ if terminal {
+ return nil
+ }
+ for _, backup := range []struct {
+ path string
+ exists bool
+ want string
+ }{
+ {path: path + ".fragment.old", exists: j.FragmentHadOld, want: j.FragmentOld},
+ {path: path + ".sidecar.old", exists: j.SidecarHadOld, want: j.SidecarOld},
+ } {
+ data, exists, err := readValidatedFile(backup.path, true)
+ if err != nil {
+ return err
+ }
+ if exists != backup.exists || (exists && digest(data) != backup.want) {
+ return fmt.Errorf("linechain journal backup shape does not match authority")
+ }
+ }
+ return nil
+}
+
+func journalTerminal(phase string) bool {
+ return phase == "terminal_desired" || phase == "terminal_old"
+}
+
+func resultSHA(result *model.TaskResult) string {
+ if result == nil {
+ return ""
+ }
+ b, err := json.Marshal(result)
+ if err != nil {
+ return ""
+ }
+ return digest(b)
+}
+
+func journalSHA(j journal) string {
+ b, err := json.Marshal(j)
+ if err != nil {
+ return ""
+ }
+ return digest(b)
+}
+
+func (m *Manager) validateDocument(d Document) error {
+ if d.Version != 2 {
+ return fmt.Errorf("unsupported linechain document version %d", d.Version)
+ }
+ if d.DurableProtocol != "linechain-e3-v2" {
+ return fmt.Errorf("unsupported linechain durable protocol %q", d.DurableProtocol)
+ }
+ if d.FragmentBasename == "" {
+ return fmt.Errorf("v2 fragment_basename is required")
+ }
+ switch d.Operation {
+ case "create":
+ if d.PreviousFragmentSHA256 != nil || d.Fragment == nil || d.FragmentSHA256 == nil || d.SidecarPatch.DesiredDownstreamLineUUID == nil {
+ return fmt.Errorf("create document has inconsistent old/desired shape")
+ }
+ case "replace":
+ if d.PreviousFragmentSHA256 == nil || d.Fragment == nil || d.FragmentSHA256 == nil || d.SidecarPatch.ExpectedDownstreamLineUUID == nil || d.SidecarPatch.DesiredDownstreamLineUUID == nil {
+ return fmt.Errorf("replace document has inconsistent old/desired shape")
+ }
+ case "remove":
+ if d.PreviousFragmentSHA256 == nil || d.Fragment != nil || d.FragmentSHA256 != nil || d.SidecarPatch.ExpectedDownstreamLineUUID == nil || d.SidecarPatch.DesiredDownstreamLineUUID != nil {
+ return fmt.Errorf("remove document has inconsistent old/desired shape")
+ }
+ default:
+ return fmt.Errorf("unsupported linechain operation %q", d.Operation)
+ }
+ binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: d.Operation, FragmentBasename: d.FragmentBasename,
+ PreviousFragmentSHA256: d.PreviousFragmentSHA256, FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256}
+ if err := verifySemanticArtifactBinding(d.Fragment, d.SidecarPatchCanonical, binding, d.ArtifactSHA256); err != nil {
+ return err
+ }
+ configDir := filepath.Clean(d.ConfigDir)
+ if !m.Configured() {
+ return fmt.Errorf("linechain runtime layout is unresolved")
+ }
+ if configDir != m.configDir {
+ return fmt.Errorf("config_dir does not match the locally resolved sing-box directory")
+ }
+ if !filepath.IsAbs(configDir) || configDir == string(filepath.Separator) {
+ return fmt.Errorf("config_dir must be absolute and non-root")
+ }
+ for _, p := range []string{d.FragmentPath, d.SidecarPath} {
+ if !filepath.IsAbs(p) || filepath.Clean(p) == string(filepath.Separator) {
+ return fmt.Errorf("artifact path must be absolute and non-root")
+ }
+ if strings.Contains(filepath.Clean(p), "..") {
+ return fmt.Errorf("artifact path escapes its directory")
+ }
+ }
+ if filepath.Clean(d.FragmentPath) == filepath.Clean(d.SidecarPath) {
+ return fmt.Errorf("fragment and sidecar paths must differ")
+ }
+ fragmentName := filepath.Base(d.FragmentPath)
+ if filepath.Dir(filepath.Clean(d.FragmentPath)) != configDir || !linechainBasenameRE.MatchString(fragmentName) {
+ return fmt.Errorf("fragment path is outside the server-owned linechain config namespace")
+ }
+ wantSidecar := m.sidecarPath
+ if filepath.Clean(d.SidecarPath) != wantSidecar {
+ return fmt.Errorf("sidecar path must match locally resolved %s", wantSidecar)
+ }
+ if err := validateParents(d.FragmentPath); err != nil {
+ return err
+ }
+ if err := validateParents(d.SidecarPath); err != nil {
+ return err
+ }
+ for _, want := range []*string{d.PreviousFragmentSHA256, d.FragmentSHA256} {
+ if want != nil && !validSHA(*want) {
+ return fmt.Errorf("previous artifact digest is invalid")
+ }
+ }
+ return nil
+}
+
+func (m *Manager) checkRestartVerify(ctx context.Context, d Document) error {
+ if _, err := m.run(ctx, m.singBoxBinary, "check", "-C", m.configDir); err != nil {
+ return fmt.Errorf("sing-box check failed")
+ }
+ restart := m.restartCommand
+ if _, err := m.run(ctx, restart[0], restart[1:]...); err != nil {
+ return fmt.Errorf("sing-box restart failed")
+ }
+ verify := m.verifyCommand
+ if _, err := m.run(ctx, verify[0], verify[1:]...); err != nil {
+ return fmt.Errorf("sing-box active verification failed")
+ }
+ return nil
+}
+
+// ResolveAfterRun converts the host transaction into one stable terminal result.
+func (m *Manager) ResolveAfterRun(ctx context.Context, task model.Task, result model.TaskResult) (model.TaskResult, error) {
+ path := m.journalPath(task.ID, task.LeaseID)
+ j, err := readJournal(path)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) && (result.ExitCode != 0 || result.Error != "") {
+ if result.FinishedAt.IsZero() {
+ result.FinishedAt = time.Now().UTC()
+ }
+ return result, nil
+ }
+ return result, err
+ }
+ if j.Result != nil {
+ return *j.Result, nil
+ }
+ if result.FinishedAt.IsZero() {
+ result.FinishedAt = time.Now().UTC()
+ }
+ if result.ExitCode == 0 && result.Error == "" && j.Phase == "desired_verified" {
+ if !pairMatches(j, true) {
+ return result, fmt.Errorf("desired linechain pair is not exact")
+ }
+ if verifyErr := m.checkRestartVerify(ctx, Document{}); verifyErr == nil {
+ j.Phase = "terminal_desired"
+ } else {
+ if err := m.restoreOld(path, &j); err != nil {
+ return result, fmt.Errorf("post-run desired runtime verification failed: %v; restore old pair: %w", verifyErr, err)
+ }
+ if err := m.checkRestartVerify(ctx, Document{}); err != nil {
+ return result, fmt.Errorf("post-run desired runtime verification failed: %v; old runtime recovery failed: %w", verifyErr, err)
+ }
+ result.ExitCode = -1
+ result.Error = "desired linechain runtime verification failed after helper return; exact old pair restored and verified"
+ j.Phase = "terminal_old"
+ }
+ } else {
+ if err := m.restoreOld(path, &j); err != nil {
+ return result, err
+ }
+ if err := m.checkRestartVerify(ctx, Document{}); err != nil {
+ return result, fmt.Errorf("verify restored old linechain runtime after helper failure: %w", err)
+ }
+ if result.ExitCode == 0 {
+ result.ExitCode = -1
+ }
+ if result.Error == "" {
+ result.Error = "linechain helper did not leave a verified desired pair; exact old pair restored"
+ }
+ j.Phase = "terminal_old"
+ }
+ j.Result = &result
+ if err := m.writeJournal(path, j); err != nil {
+ return result, err
+ }
+ return result, nil
+}
+
+// RequireRecovered resolves all non-terminal journals before network activity.
+// complete is called only with a stable exact result; cleanup follows its
+// confirmed durable completion.
+func (m *Manager) RequireRecovered(ctx context.Context, complete func(model.TaskResult) error, nodeID string) error {
+ return m.RequireRecoveredAuthorized(ctx, complete, nodeID, nil)
+}
+
+func (m *Manager) RequireRecoveredAuthorized(ctx context.Context, complete func(model.TaskResult) error, nodeID string, authority map[string]RecoveryAuthority) error {
+ entries, err := os.ReadDir(m.dir)
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ path := filepath.Join(m.dir, entry.Name())
+ var expected *RecoveryAuthority
+ if authority != nil {
+ j, err := readJournal(path)
+ if err != nil {
+ return err
+ }
+ value, ok := authority[j.TaskID+"\x00"+j.LeaseID]
+ if !ok {
+ return fmt.Errorf("linechain recovery journal is outside captured authority")
+ }
+ expected = &value
+ }
+ if err := m.recoverOne(ctx, path, complete, nodeID, "", expected); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *Manager) recoverOne(ctx context.Context, path string, complete func(model.TaskResult) error, nodeID, expectedScriptSHA string, expected ...*RecoveryAuthority) error {
+ j, err := readJournal(path)
+ if err != nil {
+ return err
+ }
+ if err := m.validateJournal(path, j); err != nil {
+ return err
+ }
+ if expectedScriptSHA != "" && j.TaskScriptSHA != expectedScriptSHA {
+ return fmt.Errorf("linechain retry task script does not match journal authority")
+ }
+ if len(expected) > 0 && expected[0] != nil {
+ want := expected[0]
+ if journalSHA(j) != want.JournalSHA256 || j.TaskScriptSHA != want.TaskScriptSHA || j.ArtifactSHA256 != want.ArtifactSHA256 || j.Phase != want.Phase || resultSHA(j.Result) != want.ResultSHA256 {
+ return fmt.Errorf("linechain journal changed after authority capture")
+ }
+ }
+ if j.Result == nil {
+ result := model.TaskResult{TaskID: j.TaskID, LeaseID: j.LeaseID, NodeID: nodeID, ExitCode: -1, Error: "linechain transaction interrupted; exact old pair restored", FinishedAt: time.Now().UTC()}
+ if j.Phase == "desired_verified" && pairMatches(j, true) {
+ if verifyErr := m.checkRestartVerify(ctx, Document{}); verifyErr == nil {
+ result.ExitCode = 0
+ result.Error = ""
+ j.Phase = "terminal_desired"
+ } else {
+ if err := m.restoreOld(path, &j); err != nil {
+ return fmt.Errorf("recover desired runtime verification failed: %v; restore old pair: %w", verifyErr, err)
+ }
+ if err := m.checkRestartVerify(ctx, Document{}); err != nil {
+ return fmt.Errorf("recover desired runtime verification failed: %v; old runtime recovery failed: %w", verifyErr, err)
+ }
+ result.Error = "recovered desired runtime verification failed; exact old pair restored and verified"
+ j.Phase = "terminal_old"
+ }
+ } else {
+ if err := m.restoreOld(path, &j); err != nil {
+ return err
+ }
+ if err := m.checkRestartVerify(ctx, Document{}); err != nil {
+ return fmt.Errorf("recover old linechain pair service state: %w", err)
+ }
+ j.Phase = "terminal_old"
+ }
+ j.Result = &result
+ if err := m.writeJournal(path, j); err != nil {
+ return err
+ }
+ }
+ if complete != nil {
+ if err := complete(*j.Result); err != nil {
+ return err
+ }
+ return m.Cleanup(j.TaskID, j.LeaseID)
+ }
+ return nil
+}
+
+func (m *Manager) Cleanup(taskID, leaseID string) error {
+ path := m.journalPath(taskID, leaseID)
+ for _, p := range []string{path + ".fragment.old", path + ".sidecar.old", path} {
+ if err := m.remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ }
+ return m.syncDirectory(m.dir)
+}
+
+func (m *Manager) rollback(ctx context.Context, path string, j *journal, d Document, cause error) error {
+ if err := m.restoreOld(path, j); err != nil {
+ return fmt.Errorf("%v; rollback failed: %w", cause, err)
+ }
+ if err := m.checkRestartVerify(ctx, d); err != nil {
+ return fmt.Errorf("%v; exact old pair restored but service recovery failed: %v", cause, err)
+ }
+ return cause
+}
+
+func (m *Manager) restoreOld(path string, j *journal) error {
+ for _, a := range []struct {
+ dst, backup string
+ had bool
+ want string
+ }{{j.FragmentPath, path + ".fragment.old", j.FragmentHadOld, j.FragmentOld}, {j.SidecarPath, path + ".sidecar.old", j.SidecarHadOld, j.SidecarOld}} {
+ if a.had {
+ data, exists, err := readValidatedFile(a.backup, true)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ return fmt.Errorf("required linechain backup is missing")
+ }
+ if digest(data) != a.want {
+ return fmt.Errorf("backup digest mismatch")
+ }
+ s := string(data)
+ if err := m.publishFile(a.dst, &s); err != nil {
+ return err
+ }
+ } else if err := m.publishFile(a.dst, nil); err != nil {
+ return err
+ }
+ }
+ if !pairMatches(*j, false) {
+ return fmt.Errorf("restored pair digest mismatch")
+ }
+ j.Phase = "old_restored"
+ return m.writeJournal(path, *j)
+}
+
+func pairMatches(j journal, desired bool) bool {
+ a, ah, e1 := readCurrent(j.FragmentPath)
+ b, bh, e2 := readCurrent(j.SidecarPath)
+ if e1 != nil || e2 != nil {
+ return false
+ }
+ if desired {
+ return digestMaybe(a, ah) == j.FragmentOutputSHA256 && digestMaybe(b, bh) == j.SidecarOutputSHA256
+ }
+ return ah == j.FragmentHadOld && bh == j.SidecarHadOld && digestMaybe(a, ah) == j.FragmentOld && digestMaybe(b, bh) == j.SidecarOld
+}
+
+func readCurrent(path string) ([]byte, bool, error) {
+ return readValidatedFile(path, false)
+}
+
+func readValidatedFile(path string, private bool) ([]byte, bool, error) {
+ f, err := openNoFollow(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, false, nil
+ }
+ if err != nil {
+ return nil, false, err
+ }
+ defer f.Close()
+ info, err := f.Stat()
+ if err != nil {
+ return nil, false, err
+ }
+ if !info.Mode().IsRegular() || !ownedPath(info) || (private && info.Mode().Perm()&0o077 != 0) {
+ return nil, false, fmt.Errorf("linechain file is not a trusted regular file: %s", path)
+ }
+ b, err := io.ReadAll(io.LimitReader(f, maxArtifactSize+1))
+ if err == nil && len(b) > maxArtifactSize {
+ return nil, false, fmt.Errorf("linechain file exceeds %d bytes: %s", maxArtifactSize, path)
+ }
+ return b, true, err
+}
+func requirePrevious(name string, data []byte, exists bool, want string) error {
+ if want == "" {
+ if exists {
+ return fmt.Errorf("unexpected existing %s artifact", name)
+ }
+ return nil
+ }
+ if !exists || digest(data) != strings.ToLower(want) {
+ return fmt.Errorf("%s artifact does not match previous digest", name)
+ }
+ return nil
+}
+func digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
+func digestPtr(s *string) string {
+ if s == nil {
+ return ""
+ }
+ return digest([]byte(*s))
+}
+func digestMaybe(b []byte, exists bool) string {
+ if !exists {
+ return ""
+ }
+ return digest(b)
+}
+func validSHA(s string) bool {
+ b, e := hex.DecodeString(s)
+ return e == nil && len(b) == sha256.Size && s == strings.ToLower(s)
+}
+func (m *Manager) writeBackup(path string, data []byte, exists bool) error {
+ if !exists {
+ return nil
+ }
+ return writeFile(path, data)
+}
+func publish(path string, content *string) error {
+ if err := validateParents(path); err != nil {
+ return err
+ }
+ if content == nil {
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ return syncDir(filepath.Dir(path))
+ }
+ return writeFile(path, []byte(*content))
+}
+func validateParents(path string) error {
+ dir := filepath.Dir(path)
+ for {
+ info, err := os.Lstat(dir)
+ if err != nil {
+ return err
+ }
+ if (!info.IsDir() && info.Mode()&os.ModeSymlink == 0) || (dir == filepath.Dir(path) && info.Mode()&os.ModeSymlink != 0) {
+ return fmt.Errorf("artifact parent must be a real directory")
+ }
+ if dir == filepath.Dir(path) && !ownedPath(info) {
+ return fmt.Errorf("artifact parent must be owned by the agent user")
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ break
+ }
+ dir = parent
+ }
+ return nil
+}
+func writeFile(path string, data []byte) error {
+ dir := filepath.Dir(path)
+ tmp, err := os.CreateTemp(dir, ".lattice-linechain-*")
+ if err != nil {
+ return err
+ }
+ name := tmp.Name()
+ defer os.Remove(name)
+ if err = tmp.Chmod(0o600); err == nil {
+ _, err = tmp.Write(data)
+ }
+ if err == nil {
+ err = tmp.Sync()
+ }
+ if closeErr := tmp.Close(); err == nil {
+ err = closeErr
+ }
+ if err != nil {
+ return err
+ }
+ if err = os.Rename(name, path); err != nil {
+ return err
+ }
+ f, err := os.Open(path)
+ if err == nil {
+ err = f.Sync()
+ _ = f.Close()
+ }
+ if err != nil {
+ return err
+ }
+ return syncDir(dir)
+}
+func writeJSON(path string, v any) error {
+ b, err := json.Marshal(v)
+ if err != nil {
+ return err
+ }
+ return writeFile(path, b)
+}
+func readJournal(path string) (journal, error) {
+ f, err := openNoFollow(path)
+ if err != nil {
+ return journal{}, err
+ }
+ defer f.Close()
+ info, err := f.Stat()
+ if err != nil {
+ return journal{}, err
+ }
+ if !info.Mode().IsRegular() || !ownedPath(info) || info.Mode().Perm()&0o077 != 0 {
+ return journal{}, fmt.Errorf("invalid linechain journal")
+ }
+ b, err := io.ReadAll(io.LimitReader(f, maxDocumentSize+1))
+ if err != nil {
+ return journal{}, err
+ }
+ if len(b) > maxDocumentSize {
+ return journal{}, fmt.Errorf("linechain journal exceeds %d bytes", maxDocumentSize)
+ }
+ var j journal
+ dec := json.NewDecoder(bytes.NewReader(b))
+ dec.DisallowUnknownFields()
+ if err = dec.Decode(&j); err != nil {
+ return j, err
+ }
+ var trailing any
+ if err = dec.Decode(&trailing); !errors.Is(err, io.EOF) {
+ if err == nil {
+ return j, fmt.Errorf("linechain journal has trailing data")
+ }
+ return j, err
+ }
+ if j.Version != journalVersion || j.TaskID == "" || j.LeaseID == "" {
+ return j, fmt.Errorf("invalid linechain journal identity")
+ }
+ return j, nil
+}
+func syncDir(path string) error {
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ return f.Sync()
+}
diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go
new file mode 100644
index 0000000..3a2aeb5
--- /dev/null
+++ b/internal/linechain/manager_test.go
@@ -0,0 +1,1197 @@
+package linechain
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/LatticeNet/lattice-sdk/model"
+)
+
+func TestApplyCreateResolveAndCleanup(t *testing.T) {
+ m, dir := testManager(t)
+ configDir := filepath.Join(dir, "conf")
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ mustMkdir(t, filepath.Dir(fragmentPath))
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"credential": "sidecar-secret"})), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}`
+ doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}
+ applyDoc(t, m, doc, "task-a", "lease-a")
+ result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()})
+ if err != nil || result.ExitCode != 0 {
+ t.Fatalf("resolve: result=%+v err=%v", result, err)
+ }
+ journalBytes, err := os.ReadFile(m.journalPath("task-a", "lease-a"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(journalBytes), fragment) || strings.Contains(string(journalBytes), "sidecar-secret") {
+ t.Fatal("journal contains credential-bearing desired bytes")
+ }
+ if err := m.Cleanup("task-a", "lease-a"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) {
+ m, dir := testManager(t)
+ mustMkdir(t, filepath.Join(dir, "conf"))
+ configDir := filepath.Join(dir, "conf")
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(fragmentPath, []byte("old"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ desired := "new"
+ doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &desired, SidecarPatch: testSidecarPatch("create")}
+ if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if err := os.Remove(fragmentPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(dir, "target"), fragmentPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := applyDocErr(m, doc, "task-b", "lease-b"); err == nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+}
+
+func TestCheckFailureRestoresExactOldPair(t *testing.T) {
+ m, dir := testManager(t)
+ mustMkdir(t, filepath.Join(dir, "conf"))
+ configDir := filepath.Join(dir, "conf")
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("false", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ oldFragment, oldSidecar := "old-fragment", testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": "old"})
+ for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} {
+ if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ newFragment := "new-fragment"
+ doc := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")}
+ if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ assertFile(t, fragmentPath, oldFragment)
+ assertFile(t, sidecarPath, oldSidecar)
+}
+
+func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) {
+ m, dir := testManager(t)
+ mustMkdir(t, filepath.Join(dir, "conf"))
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ if err := m.ConfigureLayout(filepath.Join(dir, "conf"), sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ fragmentPath := filepath.Join(m.configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath = m.sidecarPath
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ j := journal{
+ Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath,
+ FragmentOutputSHA256: digest([]byte("new-fragment")), SidecarOutputSHA256: digest([]byte("new-sidecar")), ArtifactSHA256: digest([]byte("combined")), SidecarPatchSHA256: digest([]byte("patch")), TaskScriptSHA: digest(nil), Phase: "prepared",
+ }
+ path := m.journalPath(j.TaskID, j.LeaseID)
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ var got model.TaskResult
+ if err := m.RequireRecovered(context.Background(), func(r model.TaskResult) error { got = r; return nil }, "node-a"); err != nil {
+ t.Fatal(err)
+ }
+ if got.ExitCode == 0 || got.TaskID != j.TaskID || got.LeaseID != j.LeaseID || got.FinishedAt.IsZero() {
+ t.Fatalf("bad recovered result: %+v", got)
+ }
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("journal not cleaned: %v", err)
+ }
+}
+
+func TestOpenIsExclusive(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "txn")
+ m, err := Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ if _, err := Open(dir); err == nil {
+ t.Fatal("second manager unexpectedly acquired lock")
+ }
+ if helper, err := OpenHelper(dir); err != nil {
+ t.Fatal(err)
+ } else {
+ _ = helper.Close()
+ }
+}
+
+func TestOpenRecoversStaleLockFile(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "txn")
+ if err := os.Mkdir(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, ".lock"), []byte("stale"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ m, err := Open(dir)
+ if err != nil {
+ t.Fatalf("stale lock file blocked advisory lock: %v", err)
+ }
+ defer m.Close()
+}
+
+func TestValidateDocumentBindsOwnedPaths(t *testing.T) {
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ fragment := "{}"
+ if err := m.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil {
+ t.Fatal(err)
+ }
+ basename := "lattice-linechain-0123456789abcdef0123.json"
+ valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: m.configDir, FragmentBasename: basename, FragmentPath: filepath.Join(m.configDir, basename), SidecarPath: m.sidecarPath, Fragment: &fragment, SidecarPatch: testSidecarPatch("create")})
+ if err := m.validateDocument(valid); err != nil {
+ t.Fatal(err)
+ }
+ invalid := valid
+ invalid.FragmentPath = filepath.Join(root, "outside.json")
+ if err := m.validateDocument(invalid); err == nil {
+ t.Fatal("outside fragment accepted")
+ }
+ invalid = valid
+ invalid.SidecarPath = filepath.Join(root, "other.json")
+ if err := m.validateDocument(invalid); err == nil {
+ t.Fatal("unowned sidecar accepted")
+ }
+ invalid = valid
+ invalid.FragmentBasename = "lattice-linechain-not-hex.json"
+ invalid.FragmentPath = filepath.Join(configDir, invalid.FragmentBasename)
+ if err := m.validateDocument(invalid); err == nil {
+ t.Fatal("malformed fragment basename accepted by internal validation")
+ }
+}
+
+func TestResolvePreflightFailureWithoutJournal(t *testing.T) {
+ m, _ := testManager(t)
+ finished := time.Now().UTC()
+ input := model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: -1, Error: "document rejected before host mutation", FinishedAt: finished}
+ got, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != input {
+ t.Fatalf("preflight result changed: %+v", got)
+ }
+}
+
+func TestApplyRejectsTrailingAndLegacyWireFields(t *testing.T) {
+ m, dir := testManager(t)
+ configDir := filepath.Join(dir, "conf")
+ mustMkdir(t, configDir)
+ if err := m.ConfigureLayout(configDir, filepath.Join(dir, "lattice-metadata.json")); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{}`
+ d := BindDocument(Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")})
+ base, _ := json.Marshal(wireDocumentV2{Version: d.Version, DurableProtocol: d.DurableProtocol, Operation: d.Operation, FragmentBasename: d.FragmentBasename,
+ Fragment: d.Fragment, SidecarPatch: d.SidecarPatch, PreviousFragmentSHA256: d.PreviousFragmentSHA256,
+ FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256, ArtifactSHA256: d.ArtifactSHA256})
+ for name, raw := range map[string]string{
+ "trailing": string(base) + ` {}`,
+ "duplicate known field": strings.Replace(string(base), `"version":2`, `"version":2,"version":2`, 1),
+ "legacy config path": strings.TrimSuffix(string(base), "}") + `,"config_dir":""}`,
+ "legacy fragment path": strings.TrimSuffix(string(base), "}") + `,"fragment_path":""}`,
+ "legacy sidecar path": strings.TrimSuffix(string(base), "}") + `,"sidecar_path":""}`,
+ "legacy previous sidecar digest": strings.TrimSuffix(string(base), "}") + `,"previous_sidecar_sha256":""}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ if err := m.Apply(context.Background(), strings.NewReader(raw), "task-"+name, "lease", digest([]byte("script"))); err == nil {
+ t.Fatal("invalid wire accepted")
+ }
+ })
+ }
+ t.Run("oversized whitespace", func(t *testing.T) {
+ raw := string(base) + strings.Repeat(" ", maxDocumentSize-len(base)+1)
+ if err := m.Apply(context.Background(), strings.NewReader(raw), "task-oversized", "lease", digest([]byte("script"))); err == nil || !strings.Contains(err.Error(), "exceeds") {
+ t.Fatalf("oversized wire error = %v", err)
+ }
+ })
+}
+
+func TestApplyRequiresStrictTaskScriptBinding(t *testing.T) {
+ m, _ := testManager(t)
+ for name, binding := range map[string]string{
+ "missing": "", "short": "abcd", "uppercase": strings.Repeat("A", 64),
+ } {
+ t.Run(name, func(t *testing.T) {
+ if err := m.Apply(context.Background(), strings.NewReader(`{}`), "task", "lease", binding); err == nil || !strings.Contains(err.Error(), "script binding") {
+ t.Fatalf("binding error=%v", err)
+ }
+ })
+ }
+}
+
+func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) {
+ m, dir := testManager(t)
+ configDir := filepath.Join(dir, "conf")
+ mustMkdir(t, configDir)
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"ordinary": map[string]any{"owner": "sb"}})), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{}`
+ d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}
+ applyDoc(t, m, d, "task-overlay", "lease-overlay")
+ b, err := os.ReadFile(sidecarPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got map[string]any
+ if err := json.Unmarshal(b, &got); err != nil {
+ t.Fatal(err)
+ }
+ if got["ordinary"] == nil || got["schema"] != "lattice.singbox-metadata.v2" {
+ t.Fatalf("overlay lost fields: %s", b)
+ }
+}
+
+func TestConfigureLayoutAcceptsTrustedAliasAndRejectsHostileTarget(t *testing.T) {
+ m, dir := testManager(t)
+ realDir := filepath.Join(dir, "real")
+ mustMkdir(t, realDir)
+ link := filepath.Join(dir, "link")
+ if err := os.Symlink(realDir, link); err != nil {
+ t.Fatal(err)
+ }
+ metaReal := filepath.Join(dir, "meta-real")
+ mustMkdir(t, metaReal)
+ metaAlias := filepath.Join(dir, "meta-alias")
+ if err := os.Symlink(metaReal, metaAlias); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(link, filepath.Join(metaAlias, "meta.json")); err != nil {
+ t.Fatalf("trusted config alias rejected: %v", err)
+ }
+ wantConfig, _ := filepath.EvalSymlinks(realDir)
+ wantMeta, _ := filepath.EvalSymlinks(metaReal)
+ if m.configDir != wantConfig || m.sidecarPath != filepath.Join(wantMeta, "meta.json") {
+ t.Fatalf("trusted aliases not canonicalized: config=%s sidecar=%s", m.configDir, m.sidecarPath)
+ }
+ badTarget := filepath.Join(dir, "bad-target")
+ mustMkdir(t, badTarget)
+ if err := os.Chmod(badTarget, 0o777); err != nil {
+ t.Fatal(err)
+ }
+ badAlias := filepath.Join(dir, "bad-alias")
+ if err := os.Symlink(badTarget, badAlias); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(badAlias, filepath.Join(dir, "meta.json")); err == nil {
+ t.Fatal("alias to writable config root accepted")
+ }
+}
+
+func TestApplyRejectsSymlinkAndInvalidCurrentSidecar(t *testing.T) {
+ for name, setup := range map[string]func(string) error{
+ "symlink": func(path string) error {
+ target := path + ".target"
+ if err := os.WriteFile(target, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[]}`), 0o600); err != nil {
+ return err
+ }
+ return os.Symlink(target, path)
+ },
+ "null": func(path string) error { return os.WriteFile(path, []byte(`null`), 0o600) },
+ "wrong schema": func(path string) error { return os.WriteFile(path, []byte(`{"schema":"v1","inbounds":[]}`), 0o600) },
+ "wrong inbounds": func(path string) error {
+ return os.WriteFile(path, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":{}}`), 0o600)
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ m, dir := testManager(t)
+ configDir := filepath.Join(dir, "conf")
+ mustMkdir(t, configDir)
+ sidecarPath := filepath.Join(dir, "lattice-metadata.json")
+ if err := setup(sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{}`
+ d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}
+ if err := applyDocErr(m, d, "task-"+name, "lease"); err == nil {
+ t.Fatal("invalid current sidecar accepted")
+ }
+ })
+ }
+}
+
+func TestApplyRejectsDuplicateSidecarAuthorityWithoutMutation(t *testing.T) {
+ const (
+ uuid = `11111111-1111-4111-8111-1111111111aa`
+ downstream = `33333333-3333-4333-8333-333333333333`
+ )
+ cases := map[string]string{
+ "root inbounds": `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"inbounds":[{"tag":"source","line_uuid":"` + uuid + `"}]}`,
+ "inbound line_uuid": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","line_uuid":"` + uuid + `"}]}`,
+ "inbound tag": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","tag":"source","line_uuid":"` + uuid + `"}]}`,
+ "inbound chain": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","chain":{"downstream_line_uuid":"` + downstream + `"},"chain":{"downstream_line_uuid":"` + downstream + `"}}]}`,
+ "chain downstream_line_uuid": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","chain":{"downstream_line_uuid":"` + downstream + `","downstream_line_uuid":"` + downstream + `"}}]}`,
+ }
+ for name, current := range cases {
+ t.Run(name, func(t *testing.T) {
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ sidecarPath := filepath.Join(root, "lattice-metadata.json")
+ if err := os.WriteFile(sidecarPath, []byte(current), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{}`
+ doc := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}
+ if err := applyDocErr(m, doc, "task-duplicate-"+name, "lease"); err == nil || !strings.Contains(err.Error(), "duplicate field") {
+ t.Fatalf("duplicate sidecar authority error = %v", err)
+ }
+ assertFile(t, sidecarPath, current)
+ if _, err := os.Lstat(filepath.Join(configDir, doc.FragmentBasename)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("fragment mutated before duplicate rejection: %v", err)
+ }
+ entries, err := os.ReadDir(m.dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, entry := range entries {
+ if strings.HasSuffix(entry.Name(), ".json") || strings.Contains(entry.Name(), ".old") {
+ t.Fatalf("journal mutation before duplicate rejection: %s", entry.Name())
+ }
+ }
+ })
+ }
+}
+
+func TestSnapshotRejectsRenamedAndDuplicateJournalIdentity(t *testing.T) {
+ for _, duplicate := range []bool{false, true} {
+ t.Run(map[bool]string{false: "renamed", true: "duplicate"}[duplicate], func(t *testing.T) {
+ m, _ := testManager(t)
+ j := journal{Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", Phase: "prepared"}
+ canonical := m.journalPath(j.TaskID, j.LeaseID)
+ if duplicate {
+ if err := writeJSON(canonical, j); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := writeJSON(filepath.Join(m.dir, "zz-renamed.json"), j); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("corrupt journal namespace accepted")
+ }
+ })
+ }
+}
+
+func TestSnapshotRejectsSymlinkFIFOOwnerAndMode(t *testing.T) {
+ t.Run("symlink", func(t *testing.T) {
+ m, _ := testManager(t)
+ target := filepath.Join(t.TempDir(), "target.json")
+ if err := os.WriteFile(target, []byte(`{"version":1,"task_id":"secret","lease_id":"secret"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(target, filepath.Join(m.dir, "symlink.json")); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("journal symlink accepted")
+ }
+ })
+ t.Run("fifo", func(t *testing.T) {
+ m, _ := testManager(t)
+ fifo := filepath.Join(m.dir, "fifo.json")
+ if err := exec.Command("mkfifo", fifo).Run(); err != nil {
+ t.Skipf("mkfifo unavailable: %v", err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("journal FIFO accepted")
+ }
+ })
+ t.Run("mode", func(t *testing.T) {
+ m, _ := testManager(t)
+ j := journal{Version: journalVersion, TaskID: "task-mode", LeaseID: "lease"}
+ path := m.journalPath(j.TaskID, j.LeaseID)
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(path, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("non-private journal mode accepted")
+ }
+ })
+ if os.Geteuid() == 0 {
+ t.Run("owner", func(t *testing.T) {
+ m, _ := testManager(t)
+ j := journal{Version: journalVersion, TaskID: "task-owner", LeaseID: "lease"}
+ path := m.journalPath(j.TaskID, j.LeaseID)
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chown(path, 65534, 65534); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("wrong-owner journal accepted")
+ }
+ })
+ }
+}
+
+func TestRecoveryPhasesRestoreOrCommitExactPair(t *testing.T) {
+ for _, tc := range []struct {
+ phase string
+ wantSuccess bool
+ }{
+ {phase: "prepared"},
+ {phase: "fragment_published"},
+ {phase: "pair_published"},
+ {phase: "desired_verified", wantSuccess: true},
+ } {
+ t.Run(tc.phase, func(t *testing.T) {
+ m, _ := testManager(t)
+ artifactRoot := t.TempDir()
+ configDir := filepath.Join(artifactRoot, "conf")
+ mustMkdir(t, configDir)
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(artifactRoot, "lattice-metadata.json")
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ fragmentPath = filepath.Join(m.configDir, filepath.Base(fragmentPath))
+ sidecarPath = m.sidecarPath
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ oldFragment := []byte("old-fragment")
+ oldSidecar := []byte(testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true}))
+ newFragment := []byte("new-fragment")
+ newSidecar := []byte(`{"inbounds":[],"schema":"lattice.singbox-metadata.v2"}` + "\n")
+ currentFragment, currentSidecar := newFragment, newSidecar
+ if tc.phase == "prepared" {
+ currentFragment, currentSidecar = oldFragment, oldSidecar
+ } else if tc.phase == "fragment_published" {
+ currentSidecar = oldSidecar
+ }
+ if err := os.WriteFile(fragmentPath, currentFragment, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(sidecarPath, currentSidecar, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ j := journal{
+ Version: journalVersion, TaskID: "task-" + tc.phase, LeaseID: "lease", FragmentPath: fragmentPath, SidecarPath: sidecarPath,
+ FragmentOld: digest(oldFragment), SidecarOld: digest(oldSidecar), FragmentOutputSHA256: digest(newFragment), SidecarOutputSHA256: digest(newSidecar),
+ ArtifactSHA256: digest(append(append(append([]byte{}, newFragment...), 0), newSidecar...)), SidecarPatchSHA256: digest([]byte("patch")), TaskScriptSHA: digest(nil),
+ FragmentHadOld: true, SidecarHadOld: true, Phase: tc.phase,
+ }
+ path := m.journalPath(j.TaskID, j.LeaseID)
+ if err := m.writeBackup(path+".fragment.old", oldFragment, true); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.writeBackup(path+".sidecar.old", oldSidecar, true); err != nil {
+ t.Fatal(err)
+ }
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ var got model.TaskResult
+ if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { got = result; return nil }, "node-a"); err != nil {
+ t.Fatal(err)
+ }
+ if (got.ExitCode == 0) != tc.wantSuccess {
+ t.Fatalf("result=%+v", got)
+ }
+ if tc.wantSuccess {
+ assertFile(t, fragmentPath, string(newFragment))
+ assertFile(t, sidecarPath, string(newSidecar))
+ } else {
+ assertFile(t, fragmentPath, string(oldFragment))
+ assertFile(t, sidecarPath, string(oldSidecar))
+ }
+ })
+ }
+}
+
+func TestJournalAuthorityRejectsCorruptSemanticsAndUnconfiguredRecovery(t *testing.T) {
+ t.Run("unconfigured empty allowed", func(t *testing.T) {
+ m, _ := testManager(t)
+ if refs, err := m.Snapshot(); err != nil || len(refs) != 0 {
+ t.Fatalf("empty unconfigured snapshot = %+v, %v", refs, err)
+ }
+ })
+ t.Run("unconfigured journal blocked", func(t *testing.T) {
+ m, _ := testManager(t)
+ j := journal{Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", Phase: "prepared"}
+ if err := writeJSON(m.journalPath(j.TaskID, j.LeaseID), j); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil || !strings.Contains(err.Error(), "before runtime layout") {
+ t.Fatalf("error=%v", err)
+ }
+ })
+ for name, mutate := range map[string]func(*journal){
+ "phase": func(j *journal) { j.Phase = "invented" },
+ "path": func(j *journal) { j.FragmentPath = filepath.Join(t.TempDir(), filepath.Base(j.FragmentPath)) },
+ "digest": func(j *journal) { j.SidecarOutputSHA256 = "bad" },
+ "result": func(j *journal) {
+ j.Phase = "terminal_desired"
+ j.Result = &model.TaskResult{TaskID: "wrong", LeaseID: j.LeaseID, FinishedAt: time.Now().UTC()}
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ m, path := appliedJournal(t, "task-"+name, "lease")
+ j, err := readJournal(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ mutate(&j)
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("corrupt journal semantics accepted")
+ }
+ })
+ }
+ t.Run("unknown field", func(t *testing.T) {
+ m, path := appliedJournal(t, "task-unknown-field", "lease")
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw = append(bytes.TrimSuffix(raw, []byte("}")), []byte(`,"unexpected":true}`)...)
+ if err := os.WriteFile(path, raw, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.Snapshot(); err == nil {
+ t.Fatal("unknown journal field accepted")
+ }
+ })
+}
+
+func TestAuthorizedRecoveryRejectsJournalSwapAfterSnapshot(t *testing.T) {
+ m, path := appliedJournal(t, "task-swap", "lease")
+ refs, err := m.Snapshot()
+ if err != nil || len(refs) != 1 {
+ t.Fatalf("snapshot=%+v err=%v", refs, err)
+ }
+ ref := refs[0]
+ authority := map[string]RecoveryAuthority{
+ ref.TaskID + "\x00" + ref.LeaseID: {TaskScriptSHA: ref.TaskScriptSHA, ArtifactSHA256: ref.ArtifactSHA256, Phase: ref.Phase, ResultSHA256: resultSHA(ref.Result), JournalSHA256: ref.JournalSHA256},
+ }
+ j, err := readJournal(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ j.ArtifactSHA256 = digest([]byte("swapped-artifact"))
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ beforeFragment, _, err := readCurrent(j.FragmentPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ called := false
+ if err := m.RequireRecoveredAuthorized(context.Background(), func(model.TaskResult) error { called = true; return nil }, "node-a", authority); err == nil || !strings.Contains(err.Error(), "changed after authority capture") {
+ t.Fatalf("error=%v", err)
+ }
+ afterFragment, _, err := readCurrent(j.FragmentPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if called || !bytes.Equal(beforeFragment, afterFragment) {
+ t.Fatal("journal swap mutated result or artifact")
+ }
+}
+
+func TestRecoveryRejectsOversizeArtifactAndBackupSymlink(t *testing.T) {
+ t.Run("oversize current", func(t *testing.T) {
+ m, dir := testManager(t)
+ configDir := filepath.Join(dir, "conf")
+ mustMkdir(t, configDir)
+ if err := m.ConfigureLayout(configDir, filepath.Join(dir, "meta.json")); err != nil {
+ t.Fatal(err)
+ }
+ path := filepath.Join(m.configDir, "lattice-linechain-0123456789abcdef0123.json")
+ if err := os.WriteFile(path, bytes.Repeat([]byte("x"), maxArtifactSize+1), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := readCurrent(path); err == nil || !strings.Contains(err.Error(), "exceeds") {
+ t.Fatalf("error=%v", err)
+ }
+ })
+ t.Run("backup symlink", func(t *testing.T) {
+ m, path := appliedJournal(t, "task-backup", "lease")
+ j, err := readJournal(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ old := []byte("old")
+ j.FragmentHadOld = true
+ j.FragmentOld = digest(old)
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ target := filepath.Join(t.TempDir(), "secret")
+ if err := os.WriteFile(target, old, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(target, path+".fragment.old"); err != nil {
+ t.Fatal(err)
+ }
+ before, err := os.ReadFile(j.FragmentPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := m.RequireRecovered(context.Background(), func(model.TaskResult) error { return nil }, "node-a"); err == nil {
+ t.Fatal("symlink backup accepted")
+ }
+ after, err := os.ReadFile(j.FragmentPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(before, after) {
+ t.Fatal("destination changed after rejected backup")
+ }
+ })
+}
+
+func TestApplyExactRetryReplaceAndRemove(t *testing.T) {
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ sidecarPath := filepath.Join(root, "meta.json")
+ if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"ordinary": "keep"})), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil {
+ t.Fatal(err)
+ }
+ runs := 0
+ m.run = func(context.Context, string, ...string) ([]byte, error) { runs++; return nil, nil }
+ basename := "lattice-linechain-0123456789abcdef0123.json"
+ fragment1 := "one"
+ create := Document{Version: 2, Operation: "create", FragmentBasename: basename, Fragment: &fragment1, SidecarPatch: testSidecarPatch("create")}
+ applyDoc(t, m, create, "create", "lease")
+ if err := m.Apply(context.Background(), bytes.NewReader(marshalDocument(t, create)), "create", "lease", digest([]byte("different-script"))); err == nil || !strings.Contains(err.Error(), "does not match") {
+ t.Fatalf("mismatched retry binding error=%v", err)
+ }
+ applyDoc(t, m, create, "create", "lease")
+ if runs != 6 {
+ t.Fatalf("exact retry reran host commands: %d", runs)
+ }
+ first, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "create", LeaseID: "lease"}, model.TaskResult{TaskID: "create", LeaseID: "lease", FinishedAt: time.Now().UTC()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ second, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "create", LeaseID: "lease"}, model.TaskResult{TaskID: "create", LeaseID: "lease", FinishedAt: time.Now().UTC().Add(time.Hour)})
+ if err != nil || !second.FinishedAt.Equal(first.FinishedAt) {
+ t.Fatalf("exact retry result drift: first=%+v second=%+v err=%v", first, second, err)
+ }
+ if err := m.Cleanup("create", "lease"); err != nil {
+ t.Fatal(err)
+ }
+
+ fragment2 := "two"
+ replace := Document{Version: 2, Operation: "replace", FragmentBasename: basename, PreviousFragmentSHA256: stringPtr(digest([]byte(fragment1))), Fragment: &fragment2, SidecarPatch: testSidecarPatch("replace")}
+ applyDoc(t, m, replace, "replace", "lease")
+ resolveSuccess(t, m, "replace", "lease")
+ remove := Document{Version: 2, Operation: "remove", FragmentBasename: basename, PreviousFragmentSHA256: stringPtr(digest([]byte(fragment2))), SidecarPatch: testSidecarPatch("remove")}
+ applyDoc(t, m, remove, "remove", "lease")
+ resolveSuccess(t, m, "remove", "lease")
+ if _, err := os.Stat(filepath.Join(m.configDir, basename)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("fragment remains: %v", err)
+ }
+ b, err := os.ReadFile(m.sidecarPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(b), `"ordinary":"keep"`) {
+ t.Fatalf("ordinary sidecar field lost: %s", b)
+ }
+}
+
+func TestCommandAndRollbackFailureMatrix(t *testing.T) {
+ for _, failAt := range []string{"check", "restart", "active"} {
+ t.Run(failAt, func(t *testing.T) {
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ sidecarPath := filepath.Join(root, "meta.json")
+ oldFragment := "old"
+ oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": "secret-not-logged"})
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil {
+ t.Fatal(err)
+ }
+ m.run = func(_ context.Context, name string, _ ...string) ([]byte, error) {
+ if name == failAt {
+ return []byte("old new secret-not-logged private-key-canary"), errors.New("failed")
+ }
+ return nil, nil
+ }
+ newFragment := "new"
+ d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")}
+ err := applyDocErr(m, d, "task-"+failAt, "lease")
+ if err == nil || strings.Contains(err.Error(), "secret-not-logged") || strings.Contains(err.Error(), "private-key-canary") {
+ t.Fatalf("failure=%v", err)
+ }
+ journalBytes, readErr := os.ReadFile(m.journalPath("task-"+failAt, "lease"))
+ if readErr == nil && (bytes.Contains(journalBytes, []byte("secret-not-logged")) || bytes.Contains(journalBytes, []byte("private-key-canary"))) {
+ t.Fatalf("journal leaked command output: %s", journalBytes)
+ }
+ assertFile(t, fragmentPath, oldFragment)
+ assertFile(t, sidecarPath, oldSidecar)
+ })
+ }
+
+ t.Run("rollback backup failure is loud", func(t *testing.T) {
+ m, path := appliedJournal(t, "task-rollback", "lease")
+ j, err := readJournal(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ j.Phase = "pair_published"
+ j.FragmentHadOld = true
+ j.FragmentOld = digest([]byte("old"))
+ if err := writeJSON(path, j); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(t.TempDir(), "missing"), path+".fragment.old"); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.RequireRecovered(context.Background(), func(model.TaskResult) error { return nil }, "node-a"); err == nil {
+ t.Fatal("rollback failure was hidden")
+ }
+ })
+}
+
+func TestJournalAndPublishFailureMatrix(t *testing.T) {
+ for _, failAt := range []string{"journal", "fragment publish", "sidecar publish"} {
+ t.Run(failAt, func(t *testing.T) {
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(root, "meta.json")
+ oldFragment := "old"
+ oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true})
+ if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ failed := false
+ m.ConfigureMutationForTest(func(path string, content *string) error {
+ shouldFail := (failAt == "fragment publish" && path == m.configDir+string(filepath.Separator)+filepath.Base(fragmentPath)) ||
+ (failAt == "sidecar publish" && path == m.sidecarPath)
+ if shouldFail && !failed {
+ failed = true
+ return errors.New("injected publish failure")
+ }
+ return publish(path, content)
+ }, func(path string, value any) error {
+ if failAt == "journal" && !failed {
+ failed = true
+ return errors.New("injected temp journal failure")
+ }
+ return writeJSON(path, value)
+ })
+ newFragment := "new"
+ d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")}
+ err := applyDocErr(m, d, "task-"+failAt, "lease")
+ if err == nil || !failed {
+ t.Fatalf("injected failure not surfaced: %v", err)
+ }
+ assertFile(t, fragmentPath, oldFragment)
+ assertFile(t, sidecarPath, oldSidecar)
+ })
+ }
+}
+
+func TestPostRunAndRecoveryRuntimeVerificationFailures(t *testing.T) {
+ newAppliedReplace := func(t *testing.T) (*Manager, string, string, string) {
+ t.Helper()
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json")
+ sidecarPath := filepath.Join(root, "meta.json")
+ oldFragment := "old"
+ oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true})
+ if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil {
+ t.Fatal(err)
+ }
+ m.run = func(context.Context, string, ...string) ([]byte, error) { return nil, nil }
+ newFragment := "new"
+ d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")}
+ applyDoc(t, m, d, "task-post-run", "lease")
+ return m, fragmentPath, sidecarPath, oldSidecar
+ }
+
+ for _, boundary := range []string{"resolve desired", "recover desired"} {
+ t.Run(boundary, func(t *testing.T) {
+ m, fragmentPath, sidecarPath, oldSidecar := newAppliedReplace(t)
+ failed := false
+ m.run = func(_ context.Context, name string, _ ...string) ([]byte, error) {
+ if name == "active" && !failed {
+ failed = true
+ return nil, errors.New("inactive")
+ }
+ return nil, nil
+ }
+ if boundary == "resolve desired" {
+ result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-post-run", LeaseID: "lease"}, model.TaskResult{TaskID: "task-post-run", LeaseID: "lease", FinishedAt: time.Now().UTC()})
+ if err != nil || result.ExitCode == 0 || !strings.Contains(result.Error, "exact old pair restored") {
+ t.Fatalf("result=%+v err=%v", result, err)
+ }
+ } else {
+ var result model.TaskResult
+ if err := m.RequireRecovered(context.Background(), func(got model.TaskResult) error { result = got; return nil }, "node-a"); err != nil {
+ t.Fatal(err)
+ }
+ if result.ExitCode == 0 || !strings.Contains(result.Error, "exact old pair restored") {
+ t.Fatalf("result=%+v", result)
+ }
+ }
+ assertFile(t, fragmentPath, "old")
+ assertFile(t, sidecarPath, oldSidecar)
+ })
+ }
+
+ t.Run("runner failure with inactive restored runtime remains nonterminal", func(t *testing.T) {
+ m, fragmentPath, sidecarPath, oldSidecar := newAppliedReplace(t)
+ m.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("inactive") }
+ result := model.TaskResult{TaskID: "task-post-run", LeaseID: "lease", ExitCode: -1, Error: "helper killed", FinishedAt: time.Now().UTC()}
+ if _, err := m.ResolveAfterRun(context.Background(), model.Task{ID: result.TaskID, LeaseID: result.LeaseID}, result); err == nil {
+ t.Fatal("inactive restored runtime was recorded terminal")
+ }
+ assertFile(t, fragmentPath, "old")
+ assertFile(t, sidecarPath, oldSidecar)
+ j, err := readJournal(m.journalPath(result.TaskID, result.LeaseID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if journalTerminal(j.Phase) || j.Result != nil {
+ t.Fatalf("failure became terminal: %+v", j)
+ }
+ })
+}
+
+func resolveSuccess(t *testing.T, m *Manager, taskID, leaseID string) {
+ t.Helper()
+ result := model.TaskResult{TaskID: taskID, LeaseID: leaseID, FinishedAt: time.Now().UTC()}
+ result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, result)
+ if err != nil || result.ExitCode != 0 {
+ t.Fatalf("resolve result=%+v err=%v", result, err)
+ }
+ if err := m.Cleanup(taskID, leaseID); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func appliedJournal(t *testing.T, taskID, leaseID string) (*Manager, string) {
+ t.Helper()
+ m, _ := testManager(t)
+ root := t.TempDir()
+ configDir := filepath.Join(root, "conf")
+ mustMkdir(t, configDir)
+ if err := m.ConfigureLayout(configDir, filepath.Join(root, "meta.json")); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ fragment := `{}`
+ applyDoc(t, m, Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}, taskID, leaseID)
+ return m, m.journalPath(taskID, leaseID)
+}
+
+func testManager(t *testing.T) (*Manager, string) {
+ t.Helper()
+ dir := filepath.Join(t.TempDir(), "txn")
+ m, err := Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = m.Close() })
+ return m, dir
+}
+func mustMkdir(t *testing.T, p string) {
+ t.Helper()
+ if err := os.MkdirAll(p, 0o700); err != nil {
+ t.Fatal(err)
+ }
+}
+func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) {
+ t.Helper()
+ if err := applyDocErr(m, d, task, lease); err != nil {
+ t.Fatal(err)
+ }
+}
+func applyDocErr(m *Manager, d Document, task, lease string) error {
+ d = BindDocument(d)
+ if m.sidecarPath != "" {
+ if _, err := os.Lstat(m.sidecarPath); errors.Is(err, os.ErrNotExist) {
+ inbound := map[string]any{"tag": d.SidecarPatch.SourceInboundTag, "line_uuid": d.SidecarPatch.SourceLineUUID}
+ if d.SidecarPatch.ExpectedDownstreamLineUUID != nil {
+ inbound["chain"] = map[string]any{"downstream_line_uuid": *d.SidecarPatch.ExpectedDownstreamLineUUID}
+ }
+ base, _ := json.Marshal(map[string]any{"schema": semanticSidecarMetadataSchema, "inbounds": []any{inbound}})
+ if err := os.WriteFile(m.sidecarPath, base, 0o600); err != nil {
+ return err
+ }
+ }
+ }
+ b := marshalDocument(nil, d)
+ return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease, digest([]byte(task)))
+}
+
+func testSidecarPatch(operation string) SidecarPatchV2 {
+ patch := SidecarPatchV2{Schema: semanticSidecarPatchSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", DesiredDownstreamLineUUID: stringPtr(newUUID)}
+ if operation != "create" {
+ patch.ExpectedDownstreamLineUUID = stringPtr(newUUID)
+ }
+ if operation == "remove" {
+ patch.DesiredDownstreamLineUUID = nil
+ }
+ return patch
+}
+
+func marshalDocument(t *testing.T, d Document) []byte {
+ d = BindDocument(d)
+ b, err := json.Marshal(wireDocumentV2{
+ Version: d.Version, DurableProtocol: d.DurableProtocol, Operation: d.Operation, FragmentBasename: d.FragmentBasename,
+ Fragment: d.Fragment, SidecarPatch: d.SidecarPatch, PreviousFragmentSHA256: d.PreviousFragmentSHA256,
+ FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256, ArtifactSHA256: d.ArtifactSHA256,
+ })
+ if err != nil && t != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func TestConsumeProductionServerFixture(t *testing.T) {
+ fixturePath := os.Getenv("LATTICE_LINECHAIN_SERVER_FIXTURE")
+ if fixturePath == "" {
+ t.Skip("LATTICE_LINECHAIN_SERVER_FIXTURE is not set")
+ }
+ type fixture struct {
+ Schema string `json:"schema"`
+ ApprovalArtifactSHA256 string `json:"approval_artifact_sha256"`
+ RequestSHA256 string `json:"request_sha256"`
+ TaskScriptSHA256 string `json:"task_script_sha256"`
+ TaskID string `json:"task_id"`
+ LeaseID string `json:"lease_id"`
+ Document json.RawMessage `json:"document"`
+ }
+ rawFixture, err := os.ReadFile(fixturePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var f fixture
+ dec := json.NewDecoder(bytes.NewReader(rawFixture))
+ dec.DisallowUnknownFields()
+ if err := dec.Decode(&f); err != nil {
+ t.Fatal(err)
+ }
+ if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) {
+ t.Fatalf("fixture has trailing data: %v", err)
+ }
+ if f.Schema != "lattice.linechain.cross-contract-fixture.v2" || !validSHA(f.ApprovalArtifactSHA256) || !validSHA(f.RequestSHA256) || !validSHA(f.TaskScriptSHA256) || f.TaskID == "" || f.LeaseID == "" {
+ t.Fatalf("invalid production fixture wrapper: %+v", f)
+ }
+ var document wireDocumentV2
+ if err := json.Unmarshal(f.Document, &document); err != nil {
+ t.Fatal(err)
+ }
+ if document.ArtifactSHA256 != f.ApprovalArtifactSHA256 {
+ t.Fatalf("approval/document artifact mismatch: %s != %s", f.ApprovalArtifactSHA256, document.ArtifactSHA256)
+ }
+
+ root := t.TempDir()
+ m, err := Open(filepath.Join(root, "txn"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = m.Close() })
+ configDir := filepath.Join(root, "conf")
+ if err := os.Mkdir(configDir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ sidecarPath := filepath.Join(root, "lattice-metadata.json")
+ if err := m.ConfigureLayout(configDir, sidecarPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil {
+ t.Fatal(err)
+ }
+ current := []byte("{\n \"unknown_root\": {\"preserve\": true},\n \"schema\": \"lattice.singbox-metadata.v2\",\n \"inbounds\": [\n" +
+ " {\"tag\":\"before\",\"line_uuid\":\"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa\",\"keep\":1},\n" +
+ " {\"tag\":\"source-b\",\"line_uuid\":\"22222222-2222-4222-8222-222222222222\",\"ordinary\":\"keep\"},\n" +
+ " {\"tag\":\"after\",\"line_uuid\":\"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb\",\"keep\":2}\n ]\n}\n")
+ if err := os.WriteFile(sidecarPath, current, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Apply(context.Background(), bytes.NewReader(f.Document), f.TaskID, f.LeaseID, f.TaskScriptSHA256); err != nil {
+ t.Fatal(err)
+ }
+
+ j, err := readJournal(m.journalPath(f.TaskID, f.LeaseID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ fragmentBytes, err := os.ReadFile(filepath.Join(configDir, document.FragmentBasename))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sidecarBytes, err := os.ReadFile(sidecarPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if j.ArtifactSHA256 != f.ApprovalArtifactSHA256 || j.TaskScriptSHA != f.TaskScriptSHA256 || j.FragmentOutputSHA256 != digest(fragmentBytes) || j.SidecarOutputSHA256 != digest(sidecarBytes) {
+ t.Fatalf("fixture journal authority/output mismatch: %+v", j)
+ }
+ var output struct {
+ UnknownRoot map[string]bool `json:"unknown_root"`
+ Inbounds []struct {
+ Tag string `json:"tag"`
+ Ordinary string `json:"ordinary"`
+ Keep int `json:"keep"`
+ Chain map[string]any `json:"chain"`
+ } `json:"inbounds"`
+ }
+ if err := json.Unmarshal(sidecarBytes, &output); err != nil {
+ t.Fatal(err)
+ }
+ wantDownstream := ""
+ if document.SidecarPatch.DesiredDownstreamLineUUID != nil {
+ wantDownstream = *document.SidecarPatch.DesiredDownstreamLineUUID
+ }
+ if !output.UnknownRoot["preserve"] || len(output.Inbounds) != 3 || output.Inbounds[0].Tag != "before" || output.Inbounds[0].Keep != 1 || output.Inbounds[1].Tag != "source-b" || output.Inbounds[1].Ordinary != "keep" || output.Inbounds[2].Tag != "after" || output.Inbounds[2].Keep != 2 || output.Inbounds[1].Chain["downstream_line_uuid"] != wantDownstream {
+ t.Fatalf("fixture merge did not preserve scoped host state: %s", sidecarBytes)
+ }
+
+ m.publishFile = func(string, *string) error { return errors.New("replay attempted to republish output") }
+ if err := m.Apply(context.Background(), bytes.NewReader(f.Document), f.TaskID, f.LeaseID, f.TaskScriptSHA256); err != nil {
+ t.Fatal(err)
+ }
+ replayed, err := readJournal(m.journalPath(f.TaskID, f.LeaseID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if replayed.Phase != "terminal_desired" || replayed.FragmentOutputSHA256 != j.FragmentOutputSHA256 || replayed.SidecarOutputSHA256 != j.SidecarOutputSHA256 {
+ t.Fatalf("fixture replay did not use journaled output authority: %+v", replayed)
+ }
+}
+
+func testCurrentSidecar(expected *string, extras map[string]any) string {
+ inbound := map[string]any{"tag": "source", "line_uuid": sourceUUID}
+ if expected != nil {
+ inbound["chain"] = map[string]any{"downstream_line_uuid": *expected}
+ }
+ top := map[string]any{"schema": semanticSidecarMetadataSchema, "inbounds": []any{inbound}}
+ for key, value := range extras {
+ top[key] = value
+ }
+ raw, err := json.Marshal(top)
+ if err != nil {
+ panic(err)
+ }
+ return string(raw)
+}
+
+func assertFile(t *testing.T, p, want string) {
+ t.Helper()
+ b, err := os.ReadFile(p)
+ if err != nil || string(b) != want {
+ t.Fatalf("file %s = %q err=%v", p, b, err)
+ }
+}
diff --git a/internal/linechain/read_nofollow_other.go b/internal/linechain/read_nofollow_other.go
new file mode 100644
index 0000000..45d735c
--- /dev/null
+++ b/internal/linechain/read_nofollow_other.go
@@ -0,0 +1,26 @@
+//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris
+
+package linechain
+
+import (
+ "fmt"
+ "os"
+)
+
+func openNoFollow(path string) (*os.File, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ opened, err := f.Stat()
+ if err != nil {
+ _ = f.Close()
+ return nil, err
+ }
+ visible, err := os.Lstat(path)
+ if err != nil || visible.Mode()&os.ModeSymlink != 0 || !os.SameFile(opened, visible) {
+ _ = f.Close()
+ return nil, fmt.Errorf("path changed or is a symbolic link")
+ }
+ return f, nil
+}
diff --git a/internal/linechain/read_nofollow_unix.go b/internal/linechain/read_nofollow_unix.go
new file mode 100644
index 0000000..b542db2
--- /dev/null
+++ b/internal/linechain/read_nofollow_unix.go
@@ -0,0 +1,18 @@
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package linechain
+
+import (
+ "fmt"
+ "os"
+
+ "golang.org/x/sys/unix"
+)
+
+func openNoFollow(path string) (*os.File, error) {
+ fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0)
+ if err != nil {
+ return nil, fmt.Errorf("open without following links: %w", err)
+ }
+ return os.NewFile(uintptr(fd), path), nil
+}
diff --git a/internal/linechain/sidecar_patch.go b/internal/linechain/sidecar_patch.go
new file mode 100644
index 0000000..e8bc9af
--- /dev/null
+++ b/internal/linechain/sidecar_patch.go
@@ -0,0 +1,266 @@
+package linechain
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "regexp"
+ "strings"
+)
+
+const (
+ semanticSidecarMetadataSchema = "lattice.singbox-metadata.v2"
+ semanticSidecarPatchSchema = "lattice.singbox-linechain-sidecar-patch.v1"
+ semanticArtifactSchema = "lattice.singbox-linechain-artifact.v2"
+)
+
+var lowercaseUUIDv4RE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+type SidecarPatchV2 struct {
+ Schema string `json:"schema"`
+ SourceLineUUID string `json:"source_line_uuid"`
+ SourceInboundTag string `json:"source_inbound_tag"`
+ ExpectedDownstreamLineUUID *string `json:"expected_downstream_line_uuid"`
+ DesiredDownstreamLineUUID *string `json:"desired_downstream_line_uuid"`
+}
+
+type semanticArtifactBindingV2 struct {
+ Schema string `json:"schema"`
+ Operation string `json:"operation"`
+ FragmentBasename string `json:"fragment_basename"`
+ PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"`
+ FragmentSHA256 *string `json:"fragment_sha256"`
+ SidecarPatchSHA256 string `json:"sidecar_patch_sha256"`
+}
+
+func canonicalSemanticSidecarPatch(raw []byte) (SidecarPatchV2, []byte, error) {
+ var fields map[string]json.RawMessage
+ if err := decodeJSONObject(raw, &fields); err != nil {
+ return SidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err)
+ }
+ for _, name := range []string{"schema", "source_line_uuid", "source_inbound_tag", "expected_downstream_line_uuid", "desired_downstream_line_uuid"} {
+ if _, ok := fields[name]; !ok {
+ return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch field %s must be present", name)
+ }
+ }
+ if len(fields) != 5 {
+ return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch has unknown fields")
+ }
+ var patch SidecarPatchV2
+ if err := json.Unmarshal(raw, &patch); err != nil {
+ return SidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err)
+ }
+ if patch.Schema != semanticSidecarPatchSchema || !lowercaseUUIDv4RE.MatchString(patch.SourceLineUUID) || strings.TrimSpace(patch.SourceInboundTag) == "" || patch.SourceInboundTag != strings.TrimSpace(patch.SourceInboundTag) {
+ return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch identity is invalid")
+ }
+ for _, value := range []*string{patch.ExpectedDownstreamLineUUID, patch.DesiredDownstreamLineUUID} {
+ if value != nil && !lowercaseUUIDv4RE.MatchString(*value) {
+ return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar downstream identity is invalid")
+ }
+ }
+ canonical, err := json.Marshal(patch)
+ if err != nil {
+ return SidecarPatchV2{}, nil, fmt.Errorf("encode semantic sidecar patch: %w", err)
+ }
+ if !bytes.Equal(raw, canonical) {
+ return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch is not canonical")
+ }
+ return patch, canonical, nil
+}
+
+func canonicalSemanticArtifactBinding(binding semanticArtifactBindingV2) ([]byte, error) {
+ if binding.Schema != semanticArtifactSchema || !linechainBasenameRE.MatchString(binding.FragmentBasename) || !validSHA(binding.SidecarPatchSHA256) {
+ return nil, fmt.Errorf("semantic artifact binding identity is invalid")
+ }
+ switch binding.Operation {
+ case "create":
+ if binding.PreviousFragmentSHA256 != nil || binding.FragmentSHA256 == nil {
+ return nil, fmt.Errorf("create semantic artifact binding has invalid fragment shape")
+ }
+ case "replace":
+ if binding.PreviousFragmentSHA256 == nil || binding.FragmentSHA256 == nil {
+ return nil, fmt.Errorf("replace semantic artifact binding has invalid fragment shape")
+ }
+ case "remove":
+ if binding.PreviousFragmentSHA256 == nil || binding.FragmentSHA256 != nil {
+ return nil, fmt.Errorf("remove semantic artifact binding has invalid fragment shape")
+ }
+ default:
+ return nil, fmt.Errorf("unsupported semantic artifact operation %q", binding.Operation)
+ }
+ for _, value := range []*string{binding.PreviousFragmentSHA256, binding.FragmentSHA256} {
+ if value != nil && !validSHA(*value) {
+ return nil, fmt.Errorf("semantic artifact fragment digest is invalid")
+ }
+ }
+ return json.Marshal(binding)
+}
+
+func verifySemanticArtifactBinding(fragment *string, patch []byte, binding semanticArtifactBindingV2, issuedArtifactSHA256 string) error {
+ if binding.FragmentSHA256 == nil {
+ if fragment != nil {
+ return fmt.Errorf("semantic artifact fragment output must be absent")
+ }
+ } else if fragment == nil || digest([]byte(*fragment)) != *binding.FragmentSHA256 {
+ return fmt.Errorf("semantic artifact fragment digest binding mismatch")
+ }
+ if digest(patch) != binding.SidecarPatchSHA256 {
+ return fmt.Errorf("semantic artifact sidecar patch digest binding mismatch")
+ }
+ canonical, err := canonicalSemanticArtifactBinding(binding)
+ if err != nil {
+ return err
+ }
+ if !validSHA(issuedArtifactSHA256) || digest(canonical) != issuedArtifactSHA256 {
+ return fmt.Errorf("semantic artifact digest binding mismatch")
+ }
+ return nil
+}
+
+// mergeManagedSidecar applies one authenticated source patch to an existing
+// metadata v2 sidecar. It changes only the matched inbound's chain object.
+func mergeManagedSidecar(current []byte, patch SidecarPatchV2) ([]byte, error) {
+ var top map[string]json.RawMessage
+ if err := decodeJSONObject(current, &top); err != nil {
+ return nil, fmt.Errorf("decode current semantic sidecar: %w", err)
+ }
+ if err := validateSemanticSidecarTop(top); err != nil {
+ return nil, fmt.Errorf("current semantic sidecar: %w", err)
+ }
+ var inbounds []json.RawMessage
+ if err := json.Unmarshal(top["inbounds"], &inbounds); err != nil {
+ return nil, fmt.Errorf("decode current semantic sidecar inbounds: %w", err)
+ }
+ match := -1
+ seenUUID := make(map[string]struct{}, len(inbounds))
+ seenTag := make(map[string]struct{}, len(inbounds))
+ for i, raw := range inbounds {
+ uuid, tag, err := semanticInboundIdentity(raw)
+ if err != nil {
+ return nil, fmt.Errorf("current semantic sidecar inbound %d: %w", i, err)
+ }
+ if _, ok := seenUUID[uuid]; ok {
+ return nil, fmt.Errorf("current semantic sidecar has duplicate line_uuid %q", uuid)
+ }
+ if _, ok := seenTag[tag]; ok {
+ return nil, fmt.Errorf("current semantic sidecar has duplicate tag %q", tag)
+ }
+ seenUUID[uuid], seenTag[tag] = struct{}{}, struct{}{}
+ if uuid == patch.SourceLineUUID || tag == patch.SourceInboundTag {
+ if uuid != patch.SourceLineUUID || tag != patch.SourceInboundTag || match >= 0 {
+ return nil, fmt.Errorf("current semantic sidecar source identity is ambiguous")
+ }
+ match = i
+ }
+ }
+ if match < 0 {
+ return nil, fmt.Errorf("current semantic sidecar source identity is missing")
+ }
+ var inbound map[string]json.RawMessage
+ if err := decodeJSONObject(inbounds[match], &inbound); err != nil {
+ return nil, err
+ }
+ currentDownstream, err := currentDownstreamIdentity(inbound)
+ if err != nil {
+ return nil, err
+ }
+ if !nullableStringEqual(currentDownstream, patch.ExpectedDownstreamLineUUID) {
+ return nil, fmt.Errorf("current semantic sidecar downstream identity changed")
+ }
+ if patch.DesiredDownstreamLineUUID == nil {
+ delete(inbound, "chain")
+ } else {
+ chain := struct {
+ DownstreamLineUUID string `json:"downstream_line_uuid"`
+ }{DownstreamLineUUID: *patch.DesiredDownstreamLineUUID}
+ encoded, err := json.Marshal(chain)
+ if err != nil {
+ return nil, err
+ }
+ inbound["chain"] = encoded
+ }
+ encodedInbound, err := json.Marshal(inbound)
+ if err != nil {
+ return nil, err
+ }
+ inbounds[match] = encodedInbound
+ encodedInbounds, err := json.Marshal(inbounds)
+ if err != nil {
+ return nil, err
+ }
+ top["inbounds"] = encodedInbounds
+ out, err := json.Marshal(top)
+ if err != nil {
+ return nil, err
+ }
+ return append(out, '\n'), nil
+}
+
+func currentDownstreamIdentity(inbound map[string]json.RawMessage) (*string, error) {
+ raw, ok := inbound["chain"]
+ if !ok {
+ return nil, nil
+ }
+ var chain map[string]json.RawMessage
+ if err := decodeJSONObject(raw, &chain); err != nil {
+ return nil, fmt.Errorf("current semantic sidecar chain: %w", err)
+ }
+ rawDownstream, ok := chain["downstream_line_uuid"]
+ if !ok {
+ return nil, fmt.Errorf("current semantic sidecar chain downstream_line_uuid must be present")
+ }
+ if bytes.Equal(bytes.TrimSpace(rawDownstream), []byte("null")) {
+ return nil, nil
+ }
+ var value string
+ if err := json.Unmarshal(rawDownstream, &value); err != nil || !lowercaseUUIDv4RE.MatchString(value) {
+ return nil, fmt.Errorf("current semantic sidecar downstream_line_uuid is invalid")
+ }
+ return &value, nil
+}
+
+func nullableStringEqual(a, b *string) bool {
+ if a == nil || b == nil {
+ return a == nil && b == nil
+ }
+ return *a == *b
+}
+
+func validateSemanticSidecarTop(top map[string]json.RawMessage) error {
+ var schema string
+ if err := json.Unmarshal(top["schema"], &schema); err != nil || schema != semanticSidecarMetadataSchema {
+ return fmt.Errorf("unsupported schema")
+ }
+ var inbounds []json.RawMessage
+ if err := json.Unmarshal(top["inbounds"], &inbounds); err != nil || inbounds == nil {
+ return fmt.Errorf("inbounds must be an array")
+ }
+ return nil
+}
+
+func semanticInboundIdentity(raw json.RawMessage) (string, string, error) {
+ var value map[string]json.RawMessage
+ if err := decodeJSONObject(raw, &value); err != nil {
+ return "", "", err
+ }
+ var uuid, tag string
+ if err := json.Unmarshal(value["line_uuid"], &uuid); err != nil || !lowercaseUUIDv4RE.MatchString(uuid) {
+ return "", "", fmt.Errorf("line_uuid must be a lowercase UUIDv4 string")
+ }
+ if err := json.Unmarshal(value["tag"], &tag); err != nil || strings.TrimSpace(tag) == "" || tag != strings.TrimSpace(tag) {
+ return "", "", fmt.Errorf("tag must be a non-empty canonical string")
+ }
+ return uuid, tag, nil
+}
+
+func decodeJSONObject(raw []byte, dst *map[string]json.RawMessage) error {
+ if len(bytes.TrimSpace(raw)) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
+ return fmt.Errorf("must be a non-null object")
+ }
+ fields, err := decodeUniqueJSONObject(raw)
+ if err != nil {
+ return err
+ }
+ *dst = fields
+ return nil
+}
diff --git a/internal/linechain/sidecar_patch_test.go b/internal/linechain/sidecar_patch_test.go
new file mode 100644
index 0000000..69e90f8
--- /dev/null
+++ b/internal/linechain/sidecar_patch_test.go
@@ -0,0 +1,172 @@
+package linechain
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+const (
+ sourceUUID = "11111111-1111-4111-8111-1111111111aa"
+ oldUUID = "22222222-2222-4222-8222-222222222222"
+ newUUID = "33333333-3333-4333-8333-333333333333"
+ otherUUID = "44444444-4444-4444-8444-444444444444"
+)
+
+func stringPtr(value string) *string { return &value }
+
+func patchBytes(expected, desired *string) []byte {
+ b, err := json.Marshal(SidecarPatchV2{Schema: semanticSidecarPatchSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", ExpectedDownstreamLineUUID: expected, DesiredDownstreamLineUUID: desired})
+ if err != nil {
+ panic(err)
+ }
+ return b
+}
+
+func TestCanonicalSemanticSidecarPatchRequiresExactShape(t *testing.T) {
+ raw := patchBytes(nil, stringPtr(newUUID))
+ patch, canonical, err := canonicalSemanticSidecarPatch(raw)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if patch.SourceLineUUID != sourceUUID || string(canonical) != string(raw) {
+ t.Fatalf("canonical patch mismatch: %s", canonical)
+ }
+ for _, invalid := range [][]byte{
+ []byte(`{"schema":"lattice.singbox-metadata.v2","source_line_uuid":"` + sourceUUID + `","source_inbound_tag":"source","desired_downstream_line_uuid":null}`),
+ append(append([]byte(nil), raw...), '\n'),
+ []byte(strings.Replace(string(raw), sourceUUID, strings.ToUpper(sourceUUID), 1)),
+ } {
+ if _, _, err := canonicalSemanticSidecarPatch(invalid); err == nil {
+ t.Fatalf("invalid patch accepted: %s", invalid)
+ }
+ }
+}
+
+func TestMergeManagedSidecarPreservesUnrelatedStateAndInboundFields(t *testing.T) {
+ current := []byte(`{"schema":"lattice.singbox-metadata.v2","writer":"ordinary","unknown":{"keep":true},"inbounds":[{"tag":"other","line_uuid":"` + otherUUID + `","ordinary":"keep"},{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":"` + oldUUID + `","discard":"old"},"local":"preserve"}]}`)
+ patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), stringPtr(newUUID)))
+ out, err := mergeManagedSidecar(current, patch)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got map[string]any
+ if err := json.Unmarshal(out, &got); err != nil {
+ t.Fatal(err)
+ }
+ if got["writer"] != "ordinary" || got["unknown"].(map[string]any)["keep"] != true {
+ t.Fatalf("top-level state lost: %s", out)
+ }
+ inbounds := got["inbounds"].([]any)
+ if len(inbounds) != 2 || inbounds[0].(map[string]any)["ordinary"] != "keep" || inbounds[1].(map[string]any)["local"] != "preserve" {
+ t.Fatalf("unrelated state changed: %s", out)
+ }
+ chain := inbounds[1].(map[string]any)["chain"].(map[string]any)
+ if chain["downstream_line_uuid"] != newUUID || len(chain) != 1 {
+ t.Fatalf("chain was not replaced exactly: %s", out)
+ }
+}
+
+func TestMergeManagedSidecarRemoveDeletesOnlyChain(t *testing.T) {
+ current := []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":"` + oldUUID + `"},"keep":true}]}`)
+ patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), nil))
+ out, err := mergeManagedSidecar(current, patch)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(out), "chain") || !strings.Contains(string(out), `"keep":true`) {
+ t.Fatalf("remove changed more than chain: %s", out)
+ }
+}
+
+func TestMergeManagedSidecarRejectsIdentityAndBaseDrift(t *testing.T) {
+ patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), stringPtr(newUUID)))
+ cases := []string{
+ `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + otherUUID + `"}]}`,
+ `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"one","line_uuid":"` + sourceUUID + `"},{"tag":"two","line_uuid":"` + sourceUUID + `"}]}`,
+ `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":null}}]}`,
+ `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"other","line_uuid":"` + otherUUID + `"}]}`,
+ }
+ for _, current := range cases {
+ if _, err := mergeManagedSidecar([]byte(current), patch); err == nil {
+ t.Fatalf("invalid identity/base accepted: %s", current)
+ }
+ }
+}
+
+func TestSemanticArtifactBindingNeverRewritesIssuedDigests(t *testing.T) {
+ fragment := `{"outbounds":[]}`
+ patch := patchBytes(nil, stringPtr(newUUID))
+ fragmentSHA := digest([]byte(fragment))
+ binding := semanticArtifactBindingV2{
+ Schema: semanticArtifactSchema, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json",
+ PreviousFragmentSHA256: nil, FragmentSHA256: &fragmentSHA, SidecarPatchSHA256: digest(patch),
+ }
+ canonical, err := canonicalSemanticArtifactBinding(binding)
+ if err != nil {
+ t.Fatal(err)
+ }
+ issuedArtifactSHA := digest(canonical)
+ if err := verifySemanticArtifactBinding(&fragment, patch, binding, issuedArtifactSHA); err != nil {
+ t.Fatal(err)
+ }
+ t.Run("patch", func(t *testing.T) {
+ mismatch := binding
+ mismatch.SidecarPatchSHA256 = digest([]byte("other"))
+ before := mismatch
+ if err := verifySemanticArtifactBinding(&fragment, patch, mismatch, issuedArtifactSHA); err == nil {
+ t.Fatal("patch mismatch accepted")
+ }
+ if mismatch != before {
+ t.Fatalf("issued binding rewritten: %+v -> %+v", before, mismatch)
+ }
+ })
+ for name, value := range map[string]string{
+ "fragment": digest([]byte("other")),
+ "artifact": digest([]byte("other artifact")),
+ } {
+ t.Run(name, func(t *testing.T) {
+ mismatch := binding
+ artifactSHA := issuedArtifactSHA
+ if name == "fragment" {
+ mismatch.FragmentSHA256 = &value
+ } else {
+ artifactSHA = value
+ }
+ before := mismatch
+ if err := verifySemanticArtifactBinding(&fragment, patch, mismatch, artifactSHA); err == nil {
+ t.Fatal("mismatch accepted")
+ }
+ if mismatch != before {
+ t.Fatalf("issued binding rewritten: %+v -> %+v", before, mismatch)
+ }
+ })
+ }
+}
+
+func TestCanonicalSemanticBindingMatchesServerVector(t *testing.T) {
+ const (
+ patchJSON = `{"schema":"lattice.singbox-linechain-sidecar-patch.v1","source_line_uuid":"22222222-2222-4222-8222-222222222222","source_inbound_tag":"source-b","expected_downstream_line_uuid":null,"desired_downstream_line_uuid":"11111111-1111-4111-8111-111111111111"}`
+ patchSHA = "7394c9367aa36d0e37e1e6bb70d3de70afc1d6792f56754741ba118ca2137188"
+ artifactJSON = `{"schema":"lattice.singbox-linechain-artifact.v2","operation":"create","fragment_basename":"lattice-linechain-0123456789abcdef0123.json","previous_fragment_sha256":null,"fragment_sha256":"0000000000000000000000000000000000000000000000000000000000000000","sidecar_patch_sha256":"7394c9367aa36d0e37e1e6bb70d3de70afc1d6792f56754741ba118ca2137188"}`
+ artifactSHA = "bb59094488756276a385921951eaac3e36dc604eb4a03c4cb2e1a52797aee261"
+ )
+ patch, canonicalPatch, err := canonicalSemanticSidecarPatch([]byte(patchJSON))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(canonicalPatch) != patchJSON || digest(canonicalPatch) != patchSHA {
+ t.Fatalf("server patch vector drift: bytes=%s sha=%s", canonicalPatch, digest(canonicalPatch))
+ }
+ zeroSHA := strings.Repeat("0", 64)
+ canonicalArtifact, err := canonicalSemanticArtifactBinding(semanticArtifactBindingV2{
+ Schema: semanticArtifactSchema, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json",
+ FragmentSHA256: &zeroSHA, SidecarPatchSHA256: digest(canonicalPatch),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if patch.SourceLineUUID != "22222222-2222-4222-8222-222222222222" || string(canonicalArtifact) != artifactJSON || digest(canonicalArtifact) != artifactSHA {
+ t.Fatalf("server artifact vector drift: bytes=%s sha=%s", canonicalArtifact, digest(canonicalArtifact))
+ }
+}
diff --git a/internal/singboxdiscover/discover.go b/internal/singboxdiscover/discover.go
index 309d9ac..93a9ad4 100644
--- a/internal/singboxdiscover/discover.go
+++ b/internal/singboxdiscover/discover.go
@@ -175,6 +175,14 @@ func discoverRuntimeConfig(source Source, nodeID string, at time.Time) (model.Si
return inv, nil
}
+// DiscoverRuntimeFiles parses an explicit locally resolved runtime config set.
+// It is used by recovery/E2E verification after the same bounded layout resolver
+// has established file authority.
+func DiscoverRuntimeFiles(nodeID string, files []string, metaPath string) (model.SingBoxInventory, error) {
+ copyFiles := append([]string(nil), files...)
+ return discoverRuntimeConfig(Source{MetaPath: metaPath, runtimeFiles: func() []string { return copyFiles }}, nodeID, time.Now().UTC())
+}
+
// loadSingBoxRuntimeConfigFiles locates and reads the on-box sing-box config
// files (the running process's -c/-C paths plus the /etc/sing-box defaults) and
// returns each one that parsed successfully. Returns an empty slice when none
@@ -696,6 +704,63 @@ func singBoxProcessArgs() [][]string {
return out
}
+// ResolveRuntimeLayout returns the one locally observed sing-box -C directory
+// and the design-17 sidecar path. It never trusts a server task document to
+// choose writable host paths.
+func ResolveRuntimeLayout(metaPath string) (string, string, error) {
+ return resolveRuntimeLayout(singBoxProcessArgs(), metaPath)
+}
+
+func resolveRuntimeLayout(processes [][]string, metaPath string) (string, string, error) {
+ dirs := map[string]struct{}{}
+ for _, args := range processes {
+ for i := 0; i < len(args); i++ {
+ arg := args[i]
+ var value string
+ switch arg {
+ case "-C", "--config-directory":
+ if i+1 < len(args) {
+ i++
+ value = args[i]
+ }
+ default:
+ for _, prefix := range []string{"-C=", "--config-directory="} {
+ if v, ok := strings.CutPrefix(arg, prefix); ok {
+ value = v
+ }
+ }
+ }
+ if value != "" && filepath.IsAbs(value) {
+ dirs[filepath.Clean(value)] = struct{}{}
+ }
+ }
+ }
+ if len(dirs) == 0 {
+ if info, err := os.Lstat("/etc/sing-box/conf"); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
+ dirs["/etc/sing-box/conf"] = struct{}{}
+ }
+ }
+ if len(dirs) != 1 {
+ return "", "", fmt.Errorf("resolve sing-box config directory: found %d active -C directories", len(dirs))
+ }
+ var configDir string
+ for dir := range dirs {
+ configDir = dir
+ }
+ info, err := os.Lstat(configDir)
+ if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
+ return "", "", fmt.Errorf("resolve sing-box config directory: path is not a real directory")
+ }
+ metaPath = strings.TrimSpace(metaPath)
+ if metaPath == "" {
+ metaPath = defaultMetaPath
+ }
+ if !filepath.IsAbs(metaPath) {
+ return "", "", fmt.Errorf("resolve sing-box sidecar: path must be absolute")
+ }
+ return configDir, filepath.Clean(metaPath), nil
+}
+
func containsArg(args []string, want string) bool {
for _, arg := range args {
if arg == want {
diff --git a/internal/singboxdiscover/discover_test.go b/internal/singboxdiscover/discover_test.go
index 5dab8e4..015c7dc 100644
--- a/internal/singboxdiscover/discover_test.go
+++ b/internal/singboxdiscover/discover_test.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
+ "path/filepath"
"strings"
"sync"
"sync/atomic"
@@ -50,6 +51,21 @@ func TestDiscoverParsesListAndVersion(t *testing.T) {
}
}
+func TestResolveRuntimeLayoutUsesLocalProcessAuthority(t *testing.T) {
+ dir := t.TempDir()
+ meta := filepath.Join(t.TempDir(), "lattice-metadata.json")
+ config, sidecar, err := resolveRuntimeLayout([][]string{{"/usr/bin/sing-box", "run", "-C", dir}}, meta)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if config != dir || sidecar != meta {
+ t.Fatalf("layout = %q %q", config, sidecar)
+ }
+ if _, _, err := resolveRuntimeLayout([][]string{{"sing-box", "run", "-C", dir}, {"sing-box", "run", "-C", t.TempDir()}}, meta); err == nil {
+ t.Fatal("ambiguous config directories accepted")
+ }
+}
+
func TestDiscoverListFailureReportsErrorStatus(t *testing.T) {
src := Source{
runner: func(_ context.Context, _ string, args ...string) ([]byte, error) {
diff --git a/internal/taskexec/taskexec.go b/internal/taskexec/taskexec.go
index e6c26b3..3161ac5 100644
--- a/internal/taskexec/taskexec.go
+++ b/internal/taskexec/taskexec.go
@@ -3,6 +3,8 @@ package taskexec
import (
"bytes"
"context"
+ "crypto/sha256"
+ "encoding/hex"
"errors"
"fmt"
"os"
@@ -29,8 +31,13 @@ var allowedInterpreters = map[string]string{
// because applyResourceLimits is a no-op there.
const (
// maxFileSizeBytes caps the size of any single file the task may write,
- // preventing a runaway script from filling the disk.
- maxFileSizeBytes = 8 * 1024 * 1024 // 8 MiB
+ // preventing a runaway script from filling the disk. It must still fit the
+ // largest legitimate payload a task downloads — the agent's own release
+ // binary (~tens of MiB): the 2026-08-12 fleet upgrade died to SIGXFSZ
+ // (exit 153) on 18 nodes when this was 8 MiB. The server-side update script
+ // also lifts the cap itself, so this value only guards non-update tasks and
+ // future agents whose update already ran.
+ maxFileSizeBytes = 256 * 1024 * 1024 // 256 MiB
// maxProcessHeadroom is the extra number of processes/threads a task may
// add above the agent user's current Linux-wide usage. RLIMIT_NPROC is
// scoped to the real UID rather than the task process tree, so using a fixed
@@ -163,6 +170,11 @@ type Runner struct {
// Trusted server-authored scripts use it for one-shot helper modes without
// depending on PATH or inheriting the service process environment.
AgentBinary string
+ // LinechainTxnDir is the private transaction root passed only to trusted
+ // server-authored helper invocations.
+ LinechainTxnDir string
+ LinechainConfigDir string
+ LinechainSidecarPath string
// getUID returns the effective uid of the agent process. It is a field so
// tests can simulate "running as root" without actually being root. When
// nil it defaults to os.Geteuid.
@@ -287,7 +299,14 @@ func (r Runner) effectiveUID() int {
return os.Geteuid()
}
-func (r Runner) Run(task model.Task) model.TaskResult {
+func (r Runner) Run(task model.Task) model.TaskResult { return r.run(task, false) }
+
+// RunLinechain runs an already validated E3 lease with the private host-local
+// linechain authority in its environment. Callers must select this path from
+// durable protocol metadata, never from script contents.
+func (r Runner) RunLinechain(task model.Task) model.TaskResult { return r.run(task, true) }
+
+func (r Runner) run(task model.Task, trustedLinechain bool) model.TaskResult {
start := time.Now().UTC()
result := model.TaskResult{
TaskID: task.ID,
@@ -388,6 +407,16 @@ func (r Runner) Run(task model.Task) model.TaskResult {
if filepath.IsAbs(r.AgentBinary) {
cmd.Env = append(cmd.Env, "LATTICE_AGENT_BIN="+r.AgentBinary)
}
+ if trustedLinechain && filepath.IsAbs(r.LinechainTxnDir) {
+ cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TXN_DIR="+r.LinechainTxnDir)
+ }
+ if trustedLinechain && filepath.IsAbs(r.LinechainConfigDir) && filepath.IsAbs(r.LinechainSidecarPath) {
+ cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_CONFIG_DIR="+r.LinechainConfigDir, "LATTICE_LINECHAIN_SIDECAR_PATH="+r.LinechainSidecarPath)
+ }
+ if trustedLinechain {
+ sum := sha256.Sum256([]byte(task.Script))
+ cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TASK_SCRIPT_SHA256="+hex.EncodeToString(sum[:]))
+ }
var stdout, stderr cappedBuffer
stdout.limit = limit
diff --git a/internal/taskexec/taskexec_test.go b/internal/taskexec/taskexec_test.go
index 9e169c7..23f3c29 100644
--- a/internal/taskexec/taskexec_test.go
+++ b/internal/taskexec/taskexec_test.go
@@ -1,6 +1,8 @@
package taskexec
import (
+ "crypto/sha256"
+ "encoding/hex"
"os"
"path/filepath"
"runtime"
@@ -201,6 +203,31 @@ func TestRunnerPropagatesAbsoluteAgentBinary(t *testing.T) {
}
}
+func TestRunnerRestrictsLinechainEnvironmentToTrustedPath(t *testing.T) {
+ root := t.TempDir()
+ work := filepath.Join(root, "work")
+ if err := os.Mkdir(work, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ r := Runner{
+ AllowExec: true, AllowRoot: true, WorkdirRoot: work,
+ AgentBinary: "/absolute/lattice-agent", LinechainTxnDir: filepath.Join(root, "txn"),
+ LinechainConfigDir: filepath.Join(root, "conf"), LinechainSidecarPath: filepath.Join(root, "meta.json"),
+ }
+ task := model.Task{ID: "task-env", LeaseID: "lease-env", Interpreter: "sh", TimeoutSec: 10, OutputLimit: 4096,
+ Script: `printf '%s|%s|%s|%s|%s' "$LATTICE_AGENT_BIN" "$LATTICE_LINECHAIN_TXN_DIR" "$LATTICE_LINECHAIN_CONFIG_DIR" "$LATTICE_LINECHAIN_SIDECAR_PATH" "$LATTICE_LINECHAIN_TASK_SCRIPT_SHA256"`}
+ ordinary := r.Run(task)
+ if ordinary.ExitCode != 0 || ordinary.Stdout != "/absolute/lattice-agent||||" {
+ t.Fatalf("ordinary task environment leaked linechain authority: %+v", ordinary)
+ }
+ trusted := r.RunLinechain(task)
+ sum := sha256.Sum256([]byte(task.Script))
+ want := strings.Join([]string{r.AgentBinary, r.LinechainTxnDir, r.LinechainConfigDir, r.LinechainSidecarPath, hex.EncodeToString(sum[:])}, "|")
+ if trusted.ExitCode != 0 || trusted.Stdout != want {
+ t.Fatalf("trusted E3 environment = %q result=%+v, want %q", trusted.Stdout, trusted, want)
+ }
+}
+
func TestRunnerSetsPrivateTaskTempEnvironment(t *testing.T) {
r := Runner{AllowExec: true, getUID: nonRootUID}
result := r.Run(model.Task{
diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go
index 66d835a..7ba7ec9 100644
--- a/internal/taskoutbox/outbox.go
+++ b/internal/taskoutbox/outbox.go
@@ -35,6 +35,7 @@ type Entry struct {
Version int `json:"version"`
State string `json:"state"`
Task model.Task `json:"task"`
+ DurableProtocol string `json:"durable_protocol,omitempty"`
Result *model.TaskResult `json:"result,omitempty"`
ExecutionStartedAt time.Time `json:"execution_started_at"`
UpdatedAt time.Time `json:"updated_at"`
@@ -157,6 +158,15 @@ func (s *Store) Close() error {
// must not be executed again. A true result means this call published the new
// lease journal, even if a subsequent directory sync reported an error.
func (s *Store) Begin(task model.Task) (committed bool, err error) {
+ return s.begin(task, "")
+}
+
+// BeginWithProtocol records the leased delivery discriminator for recovery.
+func (s *Store) BeginWithProtocol(task model.Task, protocol string) (bool, error) {
+ return s.begin(task, protocol)
+}
+
+func (s *Store) begin(task model.Task, protocol string) (committed bool, err error) {
if strings.TrimSpace(task.ID) == "" || strings.TrimSpace(task.LeaseID) == "" {
return false, fmt.Errorf("task id and lease id are required for durable execution")
}
@@ -167,7 +177,7 @@ func (s *Store) Begin(task model.Task) (committed bool, err error) {
key := entryKey(task.ID, task.LeaseID)
for _, entry := range entries {
if entry.Task.ID == task.ID {
- if entry.key == key && reflect.DeepEqual(entry.Task, task) {
+ if entry.key == key && reflect.DeepEqual(entry.Task, task) && entry.DurableProtocol == protocol {
return false, nil
}
return false, fmt.Errorf("task %s was redelivered with a different lease or content", task.ID)
@@ -181,6 +191,7 @@ func (s *Store) Begin(task model.Task) (committed bool, err error) {
Version: entryVersion,
State: stateLeased,
Task: task,
+ DurableProtocol: protocol,
ExecutionStartedAt: now,
UpdatedAt: now,
})
@@ -194,6 +205,9 @@ func (s *Store) Complete(result model.TaskResult) (committed bool, err error) {
if err != nil {
return false, err
}
+ if entry.State == stateDone && entry.Result != nil && reflect.DeepEqual(*entry.Result, result) {
+ return false, nil
+ }
if entry.State != stateLeased {
return false, fmt.Errorf("task lease %s is not awaiting completion", result.TaskID)
}
@@ -236,6 +250,9 @@ func (s *Store) RecoverInterrupted(nodeID string) error {
if entry.State != stateLeased {
continue
}
+ if entry.DurableProtocol == "linechain-e3-v2" {
+ return fmt.Errorf("leased E3 task %s requires linechain journal recovery before generic outbox recovery", entry.Task.ID)
+ }
now := time.Now().UTC()
result := model.TaskResult{
TaskID: entry.Task.ID,
@@ -291,6 +308,15 @@ func (s *Store) Pending() ([]Entry, error) {
return pending, nil
}
+// Snapshot returns a bounded copy of every leased and completed entry.
+func (s *Store) Snapshot() ([]Entry, error) {
+ entries, err := s.readAll()
+ if err != nil {
+ return nil, err
+ }
+ return append([]Entry(nil), entries...), nil
+}
+
// Remove atomically unlinks an acknowledged outbox entry and syncs the
// directory so the acknowledgement survives a crash.
func (s *Store) Remove(entry Entry) error {
@@ -364,6 +390,9 @@ func (s *Store) read(key string) (Entry, error) {
if entry.Version != entryVersion || entry.Task.ID == "" || entry.Task.LeaseID == "" {
return Entry{}, fmt.Errorf("invalid task result journal: %s", path)
}
+ if strings.HasPrefix(entry.DurableProtocol, "linechain-e3-") && entry.DurableProtocol != "linechain-e3-v2" {
+ return Entry{}, fmt.Errorf("unsupported persisted linechain durable protocol %q", entry.DurableProtocol)
+ }
if key != entryKey(entry.Task.ID, entry.Task.LeaseID) {
return Entry{}, fmt.Errorf("task result journal key mismatch: %s", path)
}
diff --git a/internal/taskoutbox/outbox_test.go b/internal/taskoutbox/outbox_test.go
index 8607d7e..cee122b 100644
--- a/internal/taskoutbox/outbox_test.go
+++ b/internal/taskoutbox/outbox_test.go
@@ -395,6 +395,24 @@ func TestOpenRefusesConcurrentOwnerWithoutChangingJournal(t *testing.T) {
defer second.Close()
}
+func TestOpenRejectsPersistedLegacyLinechainProtocol(t *testing.T) {
+ dir := t.TempDir()
+ store, err := Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ task := testTask()
+ if _, err := store.BeginWithProtocol(task, "linechain-e3-v1"); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Open(dir); err == nil || !strings.Contains(err.Error(), "unsupported persisted linechain durable protocol") {
+ t.Fatalf("legacy linechain outbox error = %v", err)
+ }
+}
+
func testTask() model.Task {
return model.Task{
ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo done",
diff --git a/scripts/install.sh b/scripts/install.sh
index d2400ec..84b2d75 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -17,6 +17,7 @@ token="${LATTICE_NODE_TOKEN:-}"
bin_path="${LATTICE_AGENT_BIN:-$LATTICE_HOME/lattice-agent}"
env_path="${LATTICE_AGENT_ENV:-$LATTICE_HOME/lattice-agent.env}"
state_dir="${LATTICE_AGENT_STATE:-$LATTICE_HOME/state}"
+linechain_txn_dir="${LATTICE_LINECHAIN_TXN_DIR:-$state_dir/linechain-txn}"
service_name="${LATTICE_AGENT_SERVICE:-lattice-agent}"
run_user="${LATTICE_AGENT_RUN_USER:-}"
run_group="${LATTICE_AGENT_RUN_GROUP:-}"
@@ -83,6 +84,7 @@ if [ "$(id -u)" -ne 0 ]; then
LATTICE_TASK_CGROUP_CPU_MAX="${LATTICE_TASK_CGROUP_CPU_MAX:-}" \
LATTICE_TASK_WORK_ROOT="${LATTICE_TASK_WORK_ROOT:-}" \
LATTICE_TASK_OUTBOX_DIR="${LATTICE_TASK_OUTBOX_DIR:-}" \
+ LATTICE_LINECHAIN_TXN_DIR="$linechain_txn_dir" \
LATTICE_AGENT_ALLOW_TERMINAL="${LATTICE_AGENT_ALLOW_TERMINAL:-}" \
LATTICE_TERMINAL_TRANSPORT="${LATTICE_TERMINAL_TRANSPORT:-}" \
LATTICE_IP_MODE="${LATTICE_IP_MODE:-}" LATTICE_IP_RESOLVERS="${LATTICE_IP_RESOLVERS:-}" \
@@ -196,7 +198,7 @@ load_existing_config() {
LATTICE_AGENT_ALLOW_EXEC LATTICE_AGENT_ALLOW_ROOT_EXEC LATTICE_NO_EXEC \
LATTICE_TASK_CGROUP_ROOT LATTICE_TASK_CGROUP_MEMORY_MAX \
LATTICE_TASK_CGROUP_PIDS_MAX LATTICE_TASK_CGROUP_CPU_MAX \
- LATTICE_TASK_WORK_ROOT LATTICE_TASK_OUTBOX_DIR \
+ LATTICE_TASK_WORK_ROOT LATTICE_TASK_OUTBOX_DIR LATTICE_LINECHAIN_TXN_DIR \
LATTICE_AGENT_ALLOW_TERMINAL LATTICE_TERMINAL_TRANSPORT LATTICE_IP_MODE \
LATTICE_IP_RESOLVERS LATTICE_IP_SCRIPT LATTICE_PUBLIC_IP LATTICE_PUBLIC_IP6 \
LATTICE_SSH_ALERTS LATTICE_SINGBOX_DISCOVER LATTICE_SINGBOX_BIN \
@@ -303,6 +305,23 @@ apply_service_identity_permissions() {
chown "$run_user:$run_group" "$task_outbox_leaf" || die "cannot assign $task_outbox_leaf to $run_user:$run_group"
chmod 0700 "$task_outbox_leaf" 2>/dev/null || true
fi
+ chown "$run_user:$run_group" "$linechain_txn_dir" || die "cannot assign $linechain_txn_dir to $run_user:$run_group"
+ chmod 0700 "$linechain_txn_dir" 2>/dev/null || true
+}
+
+prepare_linechain_txn_dir() {
+ case "$linechain_txn_dir" in
+ /|/bin|/boot|/dev|/etc|/lib|/lib32|/lib64|/proc|/root|/sbin|/sys|/usr|/var)
+ die "LATTICE_LINECHAIN_TXN_DIR must not be a filesystem or system root: $linechain_txn_dir" ;;
+ /*) ;;
+ *) die "LATTICE_LINECHAIN_TXN_DIR must be an absolute path" ;;
+ esac
+ [ ! -L "$linechain_txn_dir" ] || die "LATTICE_LINECHAIN_TXN_DIR must not be a symlink: $linechain_txn_dir"
+ if [ -e "$linechain_txn_dir" ] && [ ! -d "$linechain_txn_dir" ]; then
+ die "LATTICE_LINECHAIN_TXN_DIR must be a directory: $linechain_txn_dir"
+ fi
+ mkdir -p "$linechain_txn_dir" || die "cannot create LATTICE_LINECHAIN_TXN_DIR=$linechain_txn_dir"
+ chmod 0700 "$linechain_txn_dir" 2>/dev/null || true
}
prepare_task_work_root() {
@@ -421,6 +440,7 @@ fi
mkdir -p "$state_dir"
prepare_task_work_root
prepare_task_outbox_root
+prepare_linechain_txn_dir
chmod 0750 "$LATTICE_HOME" 2>/dev/null || true
apply_service_identity_permissions
@@ -475,6 +495,7 @@ LATTICE_TASK_CGROUP_PIDS_MAX=$(quote_env "${LATTICE_TASK_CGROUP_PIDS_MAX:-64}")
LATTICE_TASK_CGROUP_CPU_MAX=$(quote_env "${LATTICE_TASK_CGROUP_CPU_MAX:-100000 100000}")
LATTICE_TASK_WORK_ROOT=$(quote_env "${LATTICE_TASK_WORK_ROOT:-}")
LATTICE_TASK_OUTBOX_DIR=$(quote_env "${LATTICE_TASK_OUTBOX_DIR:-}")
+LATTICE_LINECHAIN_TXN_DIR=$(quote_env "$linechain_txn_dir")
LATTICE_AGENT_ALLOW_TERMINAL=$(quote_env "${LATTICE_AGENT_ALLOW_TERMINAL:-0}")
LATTICE_TERMINAL_TRANSPORT=$(quote_env "${LATTICE_TERMINAL_TRANSPORT:-poll}")
LATTICE_IP_MODE=$(quote_env "${LATTICE_IP_MODE:-auto}")
@@ -567,6 +588,7 @@ EOF
LATTICE_TASK_CGROUP_CPU_MAX$(xml_escape "${LATTICE_TASK_CGROUP_CPU_MAX:-100000 100000}")
LATTICE_TASK_WORK_ROOT$(xml_escape "${LATTICE_TASK_WORK_ROOT:-}")
LATTICE_TASK_OUTBOX_DIR$(xml_escape "${LATTICE_TASK_OUTBOX_DIR:-}")
+ LATTICE_LINECHAIN_TXN_DIR$(xml_escape "$linechain_txn_dir")
LATTICE_AGENT_ALLOW_TERMINAL$(xml_escape "${LATTICE_AGENT_ALLOW_TERMINAL:-0}")
LATTICE_TERMINAL_TRANSPORT$(xml_escape "${LATTICE_TERMINAL_TRANSPORT:-poll}")
LATTICE_IP_MODE$(xml_escape "${LATTICE_IP_MODE:-auto}")
diff --git a/scripts/test-install-integrity.sh b/scripts/test-install-integrity.sh
index 0ce1714..55d6727 100755
--- a/scripts/test-install-integrity.sh
+++ b/scripts/test-install-integrity.sh
@@ -121,7 +121,12 @@ for expected in \
'chown "$run_user:$run_group" "$state_dir"' \
'LATTICE_TASK_OUTBOX_DIR="${LATTICE_TASK_OUTBOX_DIR:-}"' \
'LATTICE_TASK_OUTBOX_DIR=$(quote_env "${LATTICE_TASK_OUTBOX_DIR:-}")' \
- 'chown "$run_user:$run_group" "$task_outbox_leaf"'
+ 'chown "$run_user:$run_group" "$task_outbox_leaf"' \
+ 'LATTICE_LINECHAIN_TXN_DIR="$linechain_txn_dir"' \
+ 'LATTICE_LINECHAIN_TXN_DIR=$(quote_env "$linechain_txn_dir")' \
+ 'prepare_linechain_txn_dir' \
+ 'chmod 0700 "$linechain_txn_dir"' \
+ 'chown "$run_user:$run_group" "$linechain_txn_dir"'
do
if ! grep -Fq "$expected" "$ROOT/scripts/install.sh"; then
echo "installer non-root systemd contract missing: $expected" >&2
diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh
new file mode 100755
index 0000000..7736080
--- /dev/null
+++ b/scripts/test-linechain-e2e.sh
@@ -0,0 +1,128 @@
+#!/bin/sh
+set -eu
+
+bin="${LATTICE_SINGBOX_E2E_BIN:-}"
+[ -n "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN is required" >&2; exit 1; }
+case "$bin" in /*) ;; *) echo "LATTICE_SINGBOX_E2E_BIN must be absolute" >&2; exit 1;; esac
+[ -x "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN must be executable" >&2; exit 1; }
+version="$($bin version 2>&1)" || { echo "sing-box version failed" >&2; exit 1; }
+printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || {
+ echo "LATTICE_SINGBOX_E2E_BIN must be official sing-box 1.13.x" >&2
+ printf '%s\n' "$version" >&2
+ exit 1
+}
+
+probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")"
+e2e_root="$(mktemp -d "${TMPDIR:-/tmp}/lattice-linechain-e2e.XXXXXX")"
+runtime_root="$e2e_root/runtime"
+mkdir -m 700 "$runtime_root"
+agent_bin="$e2e_root/lattice-agent"
+agent_test_bin="$e2e_root/lattice-agent.test"
+scanner_regression() {
+ matches=$(printf '%s\n' \
+ "101 100 $bin run -C $runtime_root" \
+ "102 100 sing-box run -C $runtime_root" \
+ "103 100 $agent_test_bin --runtime-root $runtime_root" \
+ "104 100 awk -v runtime=$runtime_root" | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" '
+ (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print }
+ ')
+ [ "$(printf '%s\n' "$matches" | grep -c .)" -eq 2 ] || {
+ echo "process scanner regression failed: unexpected matches" >&2
+ printf '%s\n' "$matches" >&2
+ exit 1
+ }
+ printf '%s\n' "$matches" | grep -F "$bin run -C $runtime_root" >/dev/null || exit 1
+ printf '%s\n' "$matches" | grep -F "$agent_test_bin --runtime-root $runtime_root" >/dev/null || exit 1
+}
+if [ "${LATTICE_LINECHAIN_SCANNER_REGRESSION:-}" = 1 ]; then
+ scanner_regression
+ exit 0
+fi
+server_dir="${LATTICE_SERVER_E2E_DIR:-}"
+if [ -z "$server_dir" ]; then
+ for candidate in ../../lattice-server/.wt/worker3-task18-server ../../lattice-server; do
+ if [ -f "$candidate/go.mod" ] && grep -q 'module github.com/LatticeNet/lattice-server' "$candidate/go.mod"; then
+ server_dir="$(cd "$candidate" && pwd -P)"
+ break
+ fi
+ done
+fi
+[ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; }
+[ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; }
+# Canonical server lifecycle head for the official Task29 rerun.
+git -C "$server_dir" diff --quiet 611df86 HEAD -- . \
+ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \
+ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || {
+ echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2
+ exit 1
+}
+cleanup() {
+ status=$?
+ trap - EXIT HUP INT TERM
+ process_snapshot="$e2e_root/processes"
+ pgid_snapshot="$e2e_root/pgids"
+ ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" '
+ (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print }
+ ' >"$process_snapshot"
+ if [ -s "$process_snapshot" ]; then
+ echo "linechain E2E leaked process for runtime root $runtime_root" >&2
+ cat "$process_snapshot" >&2
+ awk '{print $2}' "$process_snapshot" | sort -n -u >"$pgid_snapshot"
+ while read -r leaked_pgid; do
+ [ -n "$leaked_pgid" ] || continue
+ kill -TERM "-$leaked_pgid" 2>/dev/null || true
+ done <"$pgid_snapshot"
+ sleep 0.2
+ while read -r leaked_pgid; do
+ [ -n "$leaked_pgid" ] || continue
+ kill -KILL "-$leaked_pgid" 2>/dev/null || true
+ done <"$pgid_snapshot"
+ sleep 0.2
+ while read -r leaked_pgid; do
+ [ -n "$leaked_pgid" ] || continue
+ if kill -0 "-$leaked_pgid" 2>/dev/null; then
+ echo "linechain E2E process group $leaked_pgid survived cleanup" >&2
+ status=1
+ fi
+ done <"$pgid_snapshot"
+ ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" '
+ (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print }
+ ' >"$process_snapshot"
+ if [ -s "$process_snapshot" ]; then
+ echo "linechain E2E runtime process scan remained non-empty" >&2
+ cat "$process_snapshot" >&2
+ status=1
+ fi
+ status=1
+ fi
+ find "$probe" "$e2e_root" -type f -delete 2>/dev/null || true
+ find "$probe" "$e2e_root" -depth -type d -exec rmdir {} \; 2>/dev/null || true
+ exit "$status"
+}
+trap cleanup EXIT HUP INT TERM
+
+# Prove the root-tagged process detector before relying on it for the real run.
+sh -c 'trap : TERM; while :; do sleep 1; done' "$runtime_root" &
+detector_pid=$!
+ps -o command= -p "$detector_pid" | grep -F "$runtime_root" | grep -F 'while :' >/dev/null || {
+ kill "$detector_pid" 2>/dev/null || true
+ wait "$detector_pid" 2>/dev/null || true
+ echo "linechain E2E leak detector self-test failed" >&2
+ exit 1
+}
+kill -KILL "$detector_pid" 2>/dev/null || true
+wait "$detector_pid" 2>/dev/null || true
+cat >"$probe/config.json" <<'JSON'
+{"inbounds":[],"outbounds":[{"type":"direct","tag":"direct"}],"route":{"final":"direct"}}
+JSON
+"$bin" check -C "$probe" >/dev/null 2>&1 || { echo "sing-box 1.13.x config check failed" >&2; exit 1; }
+printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')"
+
+go build -trimpath -o "$agent_bin" ./cmd/lattice-agent
+go test -c -tags=linechain_e2e -o "$agent_test_bin" ./cmd/lattice-agent
+(
+ cd "$server_dir"
+ LATTICE_AGENT_E2E_BIN="$agent_bin" LATTICE_AGENT_E2E_TEST_BIN="$agent_test_bin" LATTICE_SINGBOX_E2E_BIN="$bin" \
+ LATTICE_LINECHAIN_E2E_RUNTIME_ROOT="$runtime_root" \
+ go test -tags=linechain_lifecycle_e2e ./internal/server -run '^TestLineChainPersistentServerAgentLifecycleE2E$' -count=1 -v
+)