From d2666e9565196b533c762ba3312583db01eb4191 Mon Sep 17 00:00:00 2001 From: cicada <122549661+lr00rl@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:14:27 +0800 Subject: [PATCH 01/43] =?UTF-8?q?fix:=20task=20=E6=96=87=E4=BB=B6=E5=A4=A7?= =?UTF-8?q?=E5=B0=8F=E4=B8=8A=E9=99=90=208MiB=20=E2=86=92=20256MiB(?= =?UTF-8?q?=E4=BF=AE=E8=88=B0=E9=98=9F=E8=87=AA=E5=8D=87=E7=BA=A7=20SIGXFS?= =?UTF-8?q?Z)=20(#10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: raise the task file-size cap from 8 MiB to 256 MiB The 8 MiB RLIMIT_FSIZE capped every file a task script could write — including the agent's own release binary during a self-update. The 2026-08-12 fleet upgrade died to SIGXFSZ (exit 153, 'File size limit exceeded') on 18 of 20 attempted nodes. The server-side update script now lifts the inherited cap itself (the only fix that reaches agents already deployed); this raise is the belt-and-braces for non-update tasks and for future agents once their update has run. 256 MiB still bounds runaway writes. * fix: bump grpc to v1.82.1 (GO-2026-6061) govulncheck flags the sing-box stats gRPC client path under the pinned v1.72.2. Unrelated to the rlimit change; the vuln DB entry postdates the last release. protobuf rides to v1.36.11. Full suite green. --- go.mod | 12 +++---- go.sum | 60 +++++++++++++++++++---------------- internal/taskexec/taskexec.go | 9 ++++-- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 3c3da45..25485ed 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require github.com/creack/pty v1.1.24 require ( github.com/gorilla/websocket v1.5.3 - google.golang.org/grpc v1.72.2 - google.golang.org/protobuf v1.36.6 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/net v0.35.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect ) diff --git a/go.sum b/go.sum index 3a8158a..eab46a9 100644 --- a/go.sum +++ b/go.sum @@ -1,40 +1,44 @@ github.com/LatticeNet/lattice-sdk v0.2.18-0.20260722123932-4a318f246d23 h1:2qpbnG8jO9lOlVRpp4OCeami34HUyV7VGmp3vj4IRQI= github.com/LatticeNet/lattice-sdk v0.2.18-0.20260722123932-4a318f246d23/go.mod h1:7ENUQ4EoS/TSW/eNomCGfZGliUPJZ46uAvp7dVcEXoE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/internal/taskexec/taskexec.go b/internal/taskexec/taskexec.go index db3fa42..c499e82 100644 --- a/internal/taskexec/taskexec.go +++ b/internal/taskexec/taskexec.go @@ -29,8 +29,13 @@ var allowedInterpreters = map[string]string{ // because applyResourceLimits is a no-op there. const ( // maxFileSizeBytes caps the size of any single file the task may write, - // preventing a runaway script from filling the disk. - maxFileSizeBytes = 8 * 1024 * 1024 // 8 MiB + // preventing a runaway script from filling the disk. It must still fit the + // largest legitimate payload a task downloads — the agent's own release + // binary (~tens of MiB): the 2026-08-12 fleet upgrade died to SIGXFSZ + // (exit 153) on 18 nodes when this was 8 MiB. The server-side update script + // also lifts the cap itself, so this value only guards non-update tasks and + // future agents whose update already ran. + maxFileSizeBytes = 256 * 1024 * 1024 // 256 MiB // maxProcessHeadroom is the extra number of processes/threads a task may // add above the agent user's current Linux-wide usage. RLIMIT_NPROC is // scoped to the real UID rather than the task process tree, so using a fixed From 1fc256fe0ce130c8b6da542556adcaf2db2d94bd Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 05:55:25 -0700 Subject: [PATCH 02/43] Prevent partial chain state from escaping host recovery Apply the E3 fragment and source-side declaration through one private, crash-recoverable transaction manager. Recovery now precedes observable agent traffic, transfers one stable terminal result into the durable outbox, and retains generic task behavior. Installer and compatibility wiring make the private transaction root and alpha contract explicit, while the real sing-box harness proves client-to-source-to-target traffic. Constraint: E3 uses the existing opaque shell task channel and adds no dependency Constraint: Host state must be exactly the old pair or desired pair before result publication Rejected: Shell traps alone | cannot recover helper SIGKILL or power loss Rejected: Generic interrupted-task synthesis | could publish an unknown result before host recovery Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep linechain journal cleanup after confirmed outbox completion and before acknowledged outbox removal Tested: go test ./...; go test -race -cover ./...; go vet ./...; installer integrity; real sing-box 1.13.18 client-B-A-origin E2E Not-tested: Full server-to-agent E3 payload integration awaits the server lane exact contract --- .github/workflows/release.yml | 2 +- cmd/lattice-agent/guard_reality_test.go | 2 +- cmd/lattice-agent/linechain_e2e_test.go | 165 +++++++ cmd/lattice-agent/main.go | 128 +++++- cmd/lattice-agent/main_test.go | 98 ++++- internal/linechain/lock_other.go | 18 + internal/linechain/lock_unix.go | 40 ++ internal/linechain/manager.go | 560 ++++++++++++++++++++++++ internal/linechain/manager_test.go | 210 +++++++++ internal/taskexec/taskexec.go | 6 + internal/taskoutbox/outbox.go | 3 + scripts/install.sh | 24 +- scripts/test-install-integrity.sh | 7 +- scripts/test-linechain-e2e.sh | 10 + 14 files changed, 1250 insertions(+), 23 deletions(-) create mode 100644 cmd/lattice-agent/linechain_e2e_test.go create mode 100644 internal/linechain/lock_other.go create mode 100644 internal/linechain/lock_unix.go create mode 100644 internal/linechain/manager.go create mode 100644 internal/linechain/manager_test.go create mode 100755 scripts/test-linechain-e2e.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0eb5994..1c9e598 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} ARTIFACT: ${{ matrix.artifact }} - LATTICE_COMPAT_SERVER_MIN: v0.2.2-alpha.2 + LATTICE_COMPAT_SERVER_MIN: v0.2.2-alpha.19 LATTICE_COMPAT_DASHBOARD_MIN: v0.2.2-alpha.7 run: | VERSION="${GITHUB_REF_NAME#v}" diff --git a/cmd/lattice-agent/guard_reality_test.go b/cmd/lattice-agent/guard_reality_test.go index 8e57d6e..0bdfa18 100644 --- a/cmd/lattice-agent/guard_reality_test.go +++ b/cmd/lattice-agent/guard_reality_test.go @@ -65,7 +65,7 @@ func TestWriteGuardManagedSHAOnlyOutputsCanonicalHashOnSuccess(t *testing.T) { func TestReportedCapabilitiesAdvertiseGuardManagedSHA(t *testing.T) { got := reportedCapabilities() - if !reflect.DeepEqual(got, []string{guardManagedSHACapability}) { + if !reflect.DeepEqual(got, []string{durableTaskResultCapability, guardManagedSHACapability}) { t.Fatalf("reported capabilities = %#v", got) } } diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go new file mode 100644 index 0000000..7cb0513 --- /dev/null +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/LatticeNet/lattice-node-agent/internal/linechain" +) + +// TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. +// Normal unit runs leave the real-binary lane to that mandatory harness. +func TestLinechainRealSingBoxE2E(t *testing.T) { + bin := os.Getenv("LATTICE_SINGBOX_E2E_BIN") + if bin == "" { + return + } + root := t.TempDir() + origin := startEchoOrigin(t) + aPort, bPort, clientPort := freePort(t), freePort(t), freePort(t) + const uuidA = "11111111-1111-4111-8111-111111111111" + const uuidB = "22222222-2222-4222-8222-222222222222" + aConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":"%s"}]}],"outbounds":[{"type":"direct","tag":"direct"}],"route":{"final":"direct"}}`, aPort, uuidA) + bConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"vless","tag":"source-b","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":"%s"}]}],"outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":"%s"}],"route":{"final":"chain-to-a"}}`, bPort, uuidB, aPort, uuidA) + clientConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"socks","tag":"client","listen":"127.0.0.1","listen_port":%d}],"outbounds":[{"type":"vless","tag":"to-b","server":"127.0.0.1","server_port":%d,"uuid":"%s"}],"route":{"final":"to-b"}}`, clientPort, bPort, uuidB) + startSingBox(t, bin, root, "a", aConfig, aPort) + startSingBox(t, bin, root, "b", bConfig, bPort) + startSingBox(t, bin, root, "client", clientConfig, clientPort) + assertSOCKSEcho(t, clientPort, origin) + + conf := filepath.Join(root, "conf") + if err := os.Mkdir(conf, 0o700); err != nil { + t.Fatal(err) + } + fragmentPath := filepath.Join(conf, "lattice-linechain-e2e.json") + sidecarPath := filepath.Join(root, "lattice-metadata.json") + fragment := "{\n \"outbounds\": [{\"type\": \"direct\", \"tag\": \"linechain-direct\"}]\n}\n" + sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}\n" + doc := linechain.Document{Version: 1, ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, SingBoxBinary: bin, CheckArgs: []string{"check", "-C", conf}, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + b, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + m, err := linechain.Open(filepath.Join(root, "txn")) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.Apply(context.Background(), bytes.NewReader(b), "e2e-task", "e2e-lease"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(fragmentPath); err != nil || string(got) != fragment { + t.Fatalf("fragment mismatch: %q %v", got, err) + } +} + +func freePort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func startEchoOrigin(t *testing.T) *net.TCPAddr { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = l.Close() }) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + go func() { defer c.Close(); _, _ = io.Copy(c, c) }() + } + }() + return l.Addr().(*net.TCPAddr) +} + +func startSingBox(t *testing.T, bin, root, name, config string, port int) { + t.Helper() + path := filepath.Join(root, name+".json") + if err := os.WriteFile(path, []byte(config), 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(bin, "run", "-c", path) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 50*time.Millisecond) + if err == nil { + _ = c.Close() + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("sing-box %s did not listen: %s", name, stderr.String()) +} + +func assertSOCKSEcho(t *testing.T, socksPort int, target *net.TCPAddr) { + t.Helper() + c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", socksPort), 2*time.Second) + if err != nil { + t.Fatal(err) + } + defer c.Close() + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + if _, err = c.Write([]byte{5, 1, 0}); err != nil { + t.Fatal(err) + } + reply := make([]byte, 2) + if _, err = io.ReadFull(c, reply); err != nil || !bytes.Equal(reply, []byte{5, 0}) { + t.Fatalf("SOCKS greeting: %v %v", reply, err) + } + req := []byte{5, 1, 0, 1, 127, 0, 0, 1, 0, 0} + binary.BigEndian.PutUint16(req[8:], uint16(target.Port)) + if _, err = c.Write(req); err != nil { + t.Fatal(err) + } + head := make([]byte, 4) + if _, err = io.ReadFull(c, head); err != nil || head[1] != 0 { + t.Fatalf("SOCKS connect: %v %v", head, err) + } + n := 6 + if head[3] == 3 { + one := make([]byte, 1) + if _, err = io.ReadFull(c, one); err != nil { + t.Fatal(err) + } + n = int(one[0]) + 2 + } else if head[3] == 4 { + n = 18 + } + if _, err = io.ReadFull(c, make([]byte, n)); err != nil { + t.Fatal(err) + } + payload := []byte("lattice-linechain-e2e") + if _, err = c.Write(payload); err != nil { + t.Fatal(err) + } + got := make([]byte, len(payload)) + if _, err = io.ReadFull(c, got); err != nil || !bytes.Equal(got, payload) { + t.Fatalf("chain echo = %q err=%v", got, err) + } +} diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 635f676..cb537e2 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -27,6 +27,7 @@ import ( "github.com/LatticeNet/lattice-node-agent/internal/guardreality" "github.com/LatticeNet/lattice-node-agent/internal/hostfacts" "github.com/LatticeNet/lattice-node-agent/internal/ipdiscover" + "github.com/LatticeNet/lattice-node-agent/internal/linechain" "github.com/LatticeNet/lattice-node-agent/internal/metrics" "github.com/LatticeNet/lattice-node-agent/internal/prober" "github.com/LatticeNet/lattice-node-agent/internal/proxyusage" @@ -37,10 +38,10 @@ import ( "github.com/LatticeNet/lattice-sdk/model" ) -var version = "0.3.3" -var compatServerMin = "v0.2.2-alpha.2" +var version = "0.3.4-alpha.1" +var compatServerMin = "v0.2.2-alpha.19" var compatDashboardMin = "v0.2.2-alpha.7" -var compatChannel = "stable" +var compatChannel = "alpha" type agentCompatibility struct { ServerMin string `json:"server_min"` @@ -62,16 +63,17 @@ func compatibilityPayload() agentCompatibility { var httpClient = &http.Client{Timeout: 30 * time.Second} const ( - defaultDebugMaxLineBytes = 4096 - defaultDebugMaxBatchLines = 100 - debugSinkMaxLines = 1000 - guardRealityReportTimeout = 10 * time.Second - guardManagedSHACapability = "netguard-managed-sha-v1" - agentCapabilitiesHeader = "X-Lattice-Agent-Capabilities" + defaultDebugMaxLineBytes = 4096 + defaultDebugMaxBatchLines = 100 + debugSinkMaxLines = 1000 + guardRealityReportTimeout = 10 * time.Second + guardManagedSHACapability = "netguard-managed-sha-v1" + durableTaskResultCapability = "durable-task-result-v1" + agentCapabilitiesHeader = "X-Lattice-Agent-Capabilities" ) func reportedCapabilities() []string { - return []string{guardManagedSHACapability} + return []string{durableTaskResultCapability, guardManagedSHACapability} } type agentConfig struct { @@ -154,6 +156,7 @@ type agentConfig struct { SingBoxMeta string LogStateDir string TaskOutboxDir string + LinechainTxnDir string } type agentRuntimePayload struct { @@ -189,6 +192,7 @@ func main() { var printVersion bool var printCompat bool var printGuardManagedSHA bool + var applyLinechain bool flag.StringVar(&cfg.Server, "server", env("LATTICE_SERVER", "http://127.0.0.1:8088"), "server base URL") flag.StringVar(&cfg.NodeID, "node-id", os.Getenv("LATTICE_NODE_ID"), "node id") flag.StringVar(&cfg.Token, "token", os.Getenv("LATTICE_NODE_TOKEN"), "node enrollment token") @@ -247,6 +251,8 @@ func main() { flag.StringVar(&cfg.SingBoxMeta, "singbox-meta", env("LATTICE_SINGBOX_META", ""), "design-15 sing-box sidecar metadata path for -singbox-discover (default /etc/sing-box/lattice-metadata.json)") flag.StringVar(&cfg.LogStateDir, "log-state-dir", os.Getenv("LATTICE_LOG_STATE_DIR"), "directory for log-tail checkpoints (empty disables checkpoint persistence; sources still tail from end)") flag.StringVar(&cfg.TaskOutboxDir, "task-outbox-dir", os.Getenv("LATTICE_TASK_OUTBOX_DIR"), "base directory for durable task-result journals (default: log state dir, or the user cache directory for manual runs)") + flag.StringVar(&cfg.LinechainTxnDir, "linechain-txn-dir", os.Getenv("LATTICE_LINECHAIN_TXN_DIR"), "private directory for crash-recoverable linechain transactions") + flag.BoolVar(&applyLinechain, "linechain-apply", false, "apply one bounded linechain document from stdin and exit") flag.BoolVar(&printVersion, "version", false, "print lattice-agent version and exit") flag.BoolVar(&printCompat, "compat-json", false, "print embedded server/dashboard compatibility metadata and exit") flag.BoolVar(&printGuardManagedSHA, "guard-managed-sha", false, "print the canonical SHA-256 of the managed lattice_guard nft table and exit") @@ -267,6 +273,21 @@ func main() { } return } + if applyLinechain { + dir, err := linechainTransactionDir(cfg) + if err != nil { + log.Fatalf("linechain transaction path failed: %v", err) + } + manager, err := linechain.OpenHelper(dir) + if err != nil { + log.Fatalf("linechain transaction manager failed: %v", err) + } + defer manager.Close() + if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID")); err != nil { + log.Fatalf("linechain apply failed: %v", err) + } + return + } cfg.Debug = cfg.LocalDebug cfg.DebugMaxLineBytes = defaultDebugMaxLineBytes cfg.DebugMaxBatchLines = defaultDebugMaxBatchLines @@ -331,6 +352,18 @@ func main() { log.Fatalf("task result outbox initialization failed: %v", err) } defer taskResults.Close() + linechainDir, err := linechainTransactionDir(cfg) + if err != nil { + log.Fatalf("linechain transaction path failed: %v", err) + } + linechainManager, err := linechain.Open(linechainDir) + if err != nil { + log.Fatalf("linechain transaction manager initialization failed: %v", err) + } + defer linechainManager.Close() + if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil { + log.Fatalf("linechain recovery blocked readiness: %v", err) + } agentBinary, err := os.Executable() if err != nil { log.Fatalf("resolve lattice-agent executable failed: %v", err) @@ -381,13 +414,18 @@ func main() { runner := taskexec.Runner{ AllowExec: cfg.AllowExec, AllowRoot: cfg.AllowRoot, Cgroup: cfg.taskCgroupConfig(), - WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary, + WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary, LinechainTxnDir: linechainDir, } monitors := newMonitorManager(cfg) logTailers := newLogTailManager(cfg) ticker := time.NewTicker(cfg.Interval) defer ticker.Stop() for { + if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil { + log.Printf("linechain recovery blocked cycle: %v", err) + <-ticker.C + continue + } if agentCfg, err := fetchAgentConfig(cfg); err != nil { debugf(cfg, "agent config fetch failed: %v", err) } else { @@ -405,7 +443,7 @@ func main() { if err := reportSingBoxInventory(cfg); err != nil { log.Printf("singbox discover error: %v", err) } - if err := runTasks(cfg, runner, taskResults); err != nil { + if err := runTasks(cfg, runner, taskResults, linechainManager); err != nil { log.Printf("task poll error: %v", err) } if assigned, err := fetchMonitors(cfg); err != nil { @@ -1118,7 +1156,16 @@ type leasedAgentTask struct { DurableResult bool `json:"durable_result"` } -func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error { +func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, managers ...*linechain.Manager) error { + var manager *linechain.Manager + if len(managers) > 0 { + manager = managers[0] + } + if manager != nil { + if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil { + return err + } + } if err := outbox.RecoverInterrupted(cfg.NodeID); err != nil { return fmt.Errorf("recover interrupted task results: %w", err) } @@ -1191,6 +1238,12 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error debugf(cfg, "task start: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec) result := runner.Run(task) result.NodeID = cfg.NodeID + if manager != nil && isLinechainTask(task) { + result, err = manager.ResolveAfterRun(context.Background(), task, result) + if err != nil { + return fmt.Errorf("resolve linechain task %s: %w", task.ID, err) + } + } debugf(cfg, "task complete: id=%s exit_code=%d error=%t", task.ID, result.ExitCode, result.Error != "") completed, completeErr := outbox.Complete(result) if completeErr != nil { @@ -1203,12 +1256,22 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error if confirmErr := outbox.ConfirmDurability(); confirmErr != nil { return fmt.Errorf("%v; confirm published task result: %w", journalErr, confirmErr) } + if manager != nil && isLinechainTask(task) { + if cleanupErr := manager.Cleanup(task.ID, task.LeaseID); cleanupErr != nil { + return fmt.Errorf("%v; cleanup confirmed linechain task: %w", journalErr, cleanupErr) + } + } if flushErr := flushTaskResults(cfg, outbox); flushErr != nil { return fmt.Errorf("%v; upload confirmed task result: %w", journalErr, flushErr) } } return journalErr } + if manager != nil && isLinechainTask(task) { + if err := manager.Cleanup(task.ID, task.LeaseID); err != nil { + return fmt.Errorf("cleanup linechain task %s after durable outbox completion: %w", task.ID, err) + } + } if err := flushTaskResults(cfg, outbox); err != nil { return err } @@ -1216,6 +1279,25 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox) error return nil } +func isLinechainTask(task model.Task) bool { + return strings.Contains(task.Script, "-linechain-apply") +} + +func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { + return manager.RequireRecovered(ctx, func(result model.TaskResult) error { + committed, err := outbox.Complete(result) + if err == nil { + return nil + } + if committed { + return outbox.ConfirmDurability() + } + // An exact completed outbox may remain after a crash before transaction + // cleanup. Complete implementations treat that replay idempotently. + return err + }, nodeID) +} + func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error { pending, err := outbox.Pending() if err != nil { @@ -1253,6 +1335,26 @@ func taskResultOutboxDir(cfg agentConfig) (string, error) { return filepath.Join(base, "task-outbox", fmt.Sprintf("%x", nodeHash[:])), nil } +func linechainTransactionDir(cfg agentConfig) (string, error) { + dir := strings.TrimSpace(cfg.LinechainTxnDir) + if dir == "" { + base := strings.TrimSpace(cfg.LogStateDir) + if base == "" { + cacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + base = filepath.Join(cacheDir, "lattice-agent") + } + dir = filepath.Join(base, "linechain-txn") + } + dir = filepath.Clean(dir) + if !filepath.IsAbs(dir) || dir == string(filepath.Separator) { + return "", fmt.Errorf("linechain transaction directory must be absolute and non-root") + } + return dir, nil +} + func postAgentJSON(cfg agentConfig, path string, payload map[string]any, out any) error { return postAgentJSONContext(context.Background(), cfg, path, payload, out) } diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index c8dcbfc..639009b 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/LatticeNet/lattice-node-agent/internal/linechain" "github.com/LatticeNet/lattice-node-agent/internal/taskoutbox" "github.com/LatticeNet/lattice-sdk/model" ) @@ -38,8 +39,8 @@ func (r *countingTaskRunner) Run(task model.Task) model.TaskResult { } func TestVersionMatchesCurrentRelease(t *testing.T) { - if version != "0.3.3" { - t.Fatalf("version = %q, want 0.3.3", version) + if version != "0.3.4-alpha.1" { + t.Fatalf("version = %q, want 0.3.4-alpha.1", version) } } @@ -48,13 +49,13 @@ func TestCompatibilityPayloadIsEmbedded(t *testing.T) { if got.ServerMin == "" || got.DashboardMin == "" || got.Channel == "" { t.Fatalf("compatibility metadata must be embedded: %+v", got) } - if got.Channel != "stable" { - t.Fatalf("compatibility channel = %q, want stable", got.Channel) + if got.Channel != "alpha" { + t.Fatalf("compatibility channel = %q, want alpha", got.Channel) } // The floors stay on design-15 prerelease coordinates on purpose: no stable // server or dashboard satisfies this agent yet (stable server is still v0.2.1), // so naming a stable floor here would be a claim the ecosystem cannot back. - if got.ServerMin != "v0.2.2-alpha.2" || got.DashboardMin != "v0.2.2-alpha.7" { + if got.ServerMin != "v0.2.2-alpha.19" || got.DashboardMin != "v0.2.2-alpha.7" { t.Fatalf("compatibility floor = %+v, want coordinated design-15 alpha", got) } } @@ -258,7 +259,7 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { switch r.URL.Path { case "/api/agent/tasks": - if r.Header.Get(agentCapabilitiesHeader) != guardManagedSHACapability { + if r.Header.Get(agentCapabilitiesHeader) != strings.Join(reportedCapabilities(), ",") { return testResponse(http.StatusBadRequest, "missing lease-time capability"), nil } fetchCalls++ @@ -308,6 +309,91 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { } } +type linechainTaskRunner struct { + manager *linechain.Manager + doc []byte + calls int +} + +func (r *linechainTaskRunner) Run(task model.Task) model.TaskResult { + r.calls++ + err := r.manager.Apply(context.Background(), bytes.NewReader(r.doc), task.ID, task.LeaseID) + result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, StartedAt: time.Now().UTC(), FinishedAt: time.Now().UTC()} + if err != nil { + result.ExitCode = -1 + result.Error = err.Error() + } + return result +} + +func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { + oldClient := httpClient + t.Cleanup(func() { httpClient = oldClient }) + root := t.TempDir() + txnDir := filepath.Join(root, "txn") + manager, err := linechain.Open(txnDir) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + outbox, err := taskoutbox.Open(filepath.Join(root, "outbox")) + if err != nil { + t.Fatal(err) + } + defer outbox.Close() + conf := filepath.Join(root, "conf") + if err := os.Mkdir(conf, 0o700); err != nil { + t.Fatal(err) + } + fragmentPath := filepath.Join(conf, "lattice-linechain-chain.json") + sidecarPath := filepath.Join(root, "lattice-metadata.json") + fragment := "{\"outbounds\":[]}" + sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}" + doc, err := json.Marshal(linechain.Document{Version: 1, ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, SingBoxBinary: "true", RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}}) + if err != nil { + t.Fatal(err) + } + task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: `"$LATTICE_AGENT_BIN" -linechain-apply`, TimeoutSec: 10, OutputLimit: 1024} + runner := &linechainTaskRunner{manager: manager, doc: doc} + fetches := 0 + posts := 0 + httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/api/agent/tasks": + fetches++ + if fetches == 1 { + b, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + return testResponse(http.StatusOK, string(b)), nil + } + return testResponse(http.StatusOK, "[]"), nil + case "/api/agent/task-result": + posts++ + return testResponse(http.StatusOK, `{"ok":true}`), nil + default: + return testResponse(http.StatusNotFound, ""), nil + } + })} + cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"} + if err := runTasks(cfg, runner, outbox, manager); err != nil { + t.Fatal(err) + } + if err := runTasks(cfg, runner, outbox, manager); err != nil { + t.Fatal(err) + } + if runner.calls != 1 || posts != 1 { + t.Fatalf("runner calls=%d posts=%d", runner.calls, posts) + } + if entries, err := os.ReadDir(txnDir); err != nil { + t.Fatal(err) + } else { + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".json") { + t.Fatalf("journal not cleaned: %s", e.Name()) + } + } + } +} + func TestRunTasksRestartFlushesCompletedResultBeforeFetch(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() diff --git a/internal/linechain/lock_other.go b/internal/linechain/lock_other.go new file mode 100644 index 0000000..9455bbd --- /dev/null +++ b/internal/linechain/lock_other.go @@ -0,0 +1,18 @@ +//go:build !unix + +package linechain + +import "os" + +func lockManager(path string) (*os.File, error) { + return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) +} +func unlockManager(f *os.File) error { + name := f.Name() + err := f.Close() + if removeErr := os.Remove(name); err == nil { + err = removeErr + } + return err +} +func ownedPath(os.FileInfo) bool { return true } diff --git a/internal/linechain/lock_unix.go b/internal/linechain/lock_unix.go new file mode 100644 index 0000000..43dd7d3 --- /dev/null +++ b/internal/linechain/lock_unix.go @@ -0,0 +1,40 @@ +//go:build unix + +package linechain + +import ( + "fmt" + "os" + "syscall" +) + +func lockManager(path string) (*os.File, error) { + fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fd), path) + if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, fmt.Errorf("linechain transaction manager already open: %w", err) + } + if err := f.Chmod(0o600); err != nil { + _ = syscall.Flock(fd, syscall.LOCK_UN) + _ = f.Close() + return nil, err + } + return f, nil +} + +func unlockManager(f *os.File) error { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + if closeErr := f.Close(); err == nil { + err = closeErr + } + return err +} + +func ownedPath(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && int(stat.Uid) == os.Geteuid() +} diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go new file mode 100644 index 0000000..e69eaec --- /dev/null +++ b/internal/linechain/manager.go @@ -0,0 +1,560 @@ +// Package linechain applies the two host-local artifacts that define a +// sing-box line chain as one crash-recoverable transaction. +package linechain + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/LatticeNet/lattice-sdk/model" +) + +const ( + journalVersion = 1 + maxDocumentSize = 4 << 20 + maxDiagnostic = 4096 +) + +// Document is the bounded server-rendered input consumed by -linechain-apply. +// Desired content is never copied into the journal; only its digest is stored. +type Document struct { + Version int `json:"version"` + ConfigDir string `json:"config_dir"` + FragmentPath string `json:"fragment_path"` + SidecarPath string `json:"sidecar_path"` + PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` + PreviousSidecarSHA256 string `json:"previous_sidecar_sha256,omitempty"` + Fragment *string `json:"fragment,omitempty"` + Sidecar *string `json:"sidecar,omitempty"` + SingBoxBinary string `json:"singbox_binary,omitempty"` + CheckArgs []string `json:"check_args,omitempty"` + RestartCommand []string `json:"restart_command,omitempty"` + VerifyCommand []string `json:"verify_command,omitempty"` +} + +type journal struct { + Version int `json:"version"` + TaskID string `json:"task_id"` + LeaseID string `json:"lease_id"` + FragmentPath string `json:"fragment_path"` + SidecarPath string `json:"sidecar_path"` + FragmentOld string `json:"fragment_old_sha256,omitempty"` + SidecarOld string `json:"sidecar_old_sha256,omitempty"` + FragmentNew string `json:"fragment_desired_sha256,omitempty"` + SidecarNew string `json:"sidecar_desired_sha256,omitempty"` + FragmentHadOld bool `json:"fragment_had_old"` + SidecarHadOld bool `json:"sidecar_had_old"` + Phase string `json:"phase"` + Result *model.TaskResult `json:"result,omitempty"` +} + +type Manager struct { + dir string + lock *os.File + run func(context.Context, string, ...string) ([]byte, error) +} + +// Open takes exclusive ownership of a private absolute transaction directory. +func Open(dir string) (*Manager, error) { + m, err := open(dir) + if err != nil { + return nil, err + } + lock, err := lockManager(filepath.Join(m.dir, ".lock")) + if err != nil { + return nil, fmt.Errorf("linechain transaction manager already open: %w", err) + } + m.lock = lock + return m, syncDir(m.dir) +} + +// OpenHelper opens the transaction directory without taking the manager lock. +// It is used only by a child helper spawned by the lock-owning agent. +func OpenHelper(dir string) (*Manager, error) { return open(dir) } + +func open(dir string) (*Manager, error) { + dir = filepath.Clean(strings.TrimSpace(dir)) + if dir == "." || !filepath.IsAbs(dir) || dir == string(filepath.Separator) { + return nil, fmt.Errorf("linechain transaction directory must be absolute and non-root") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("linechain transaction path must be a real directory") + } + if err := os.Chmod(dir, 0o700); err != nil { + return nil, err + } + m := &Manager{dir: dir} + m.run = func(ctx context.Context, name string, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, name, args...).CombinedOutput() + } + return m, nil +} + +func (m *Manager) Close() error { + if m == nil || m.lock == nil { + return nil + } + err := unlockManager(m.lock) + m.lock = nil + return err +} + +func (m *Manager) journalPath(taskID, leaseID string) string { + s := sha256.Sum256([]byte(taskID + "\x00" + leaseID)) + return filepath.Join(m.dir, hex.EncodeToString(s[:])+".json") +} + +// Apply reads and applies one document. It is intended for the early-exit +// helper mode and obtains task identity from the runner's minimal environment. +func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string) error { + if strings.TrimSpace(taskID) == "" || strings.TrimSpace(leaseID) == "" { + return fmt.Errorf("task and lease identity are required") + } + dec := json.NewDecoder(io.LimitReader(r, maxDocumentSize+1)) + dec.DisallowUnknownFields() + var d Document + if err := dec.Decode(&d); err != nil { + return fmt.Errorf("decode linechain document: %w", err) + } + if err := validateDocument(d); err != nil { + return err + } + path := m.journalPath(taskID, leaseID) + if _, err := os.Lstat(path); err == nil { + return m.recoverOne(ctx, path, nil, "") + } + + fragmentOld, fragmentHad, err := readCurrent(d.FragmentPath) + if err != nil { + return err + } + sidecarOld, sidecarHad, err := readCurrent(d.SidecarPath) + if err != nil { + return err + } + if err := requirePrevious("fragment", fragmentOld, fragmentHad, d.PreviousFragmentSHA256); err != nil { + return err + } + if err := requirePrevious("sidecar", sidecarOld, sidecarHad, d.PreviousSidecarSHA256); err != nil { + return err + } + j := journal{Version: journalVersion, TaskID: taskID, LeaseID: leaseID, FragmentPath: d.FragmentPath, SidecarPath: d.SidecarPath, + FragmentOld: digest(fragmentOld), SidecarOld: digest(sidecarOld), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad, + FragmentNew: digestPtr(d.Fragment), SidecarNew: digestPtr(d.Sidecar), Phase: "prepared"} + if err := m.writeBackup(path+".fragment.old", fragmentOld, fragmentHad); err != nil { + return err + } + if err := m.writeBackup(path+".sidecar.old", sidecarOld, sidecarHad); err != nil { + return err + } + if err := writeJSON(path, j); err != nil { + return err + } + if err := publish(d.FragmentPath, d.Fragment); err != nil { + return m.rollback(ctx, path, &j, d, fmt.Errorf("publish fragment: %w", err)) + } + j.Phase = "fragment_published" + if err := writeJSON(path, j); err != nil { + return err + } + if err := publish(d.SidecarPath, d.Sidecar); err != nil { + return m.rollback(ctx, path, &j, d, fmt.Errorf("publish sidecar: %w", err)) + } + j.Phase = "pair_published" + if err := writeJSON(path, j); err != nil { + return err + } + if err := m.checkRestartVerify(ctx, d); err != nil { + return m.rollback(ctx, path, &j, d, err) + } + j.Phase = "desired_verified" + return writeJSON(path, j) +} + +func validateDocument(d Document) error { + if d.Version != journalVersion { + return fmt.Errorf("unsupported linechain document version %d", d.Version) + } + configDir := filepath.Clean(d.ConfigDir) + if !filepath.IsAbs(configDir) || configDir == string(filepath.Separator) { + return fmt.Errorf("config_dir must be absolute and non-root") + } + for _, p := range []string{d.FragmentPath, d.SidecarPath} { + if !filepath.IsAbs(p) || filepath.Clean(p) == string(filepath.Separator) { + return fmt.Errorf("artifact path must be absolute and non-root") + } + if strings.Contains(filepath.Clean(p), "..") { + return fmt.Errorf("artifact path escapes its directory") + } + } + if filepath.Clean(d.FragmentPath) == filepath.Clean(d.SidecarPath) { + return fmt.Errorf("fragment and sidecar paths must differ") + } + fragmentName := filepath.Base(d.FragmentPath) + if filepath.Dir(filepath.Clean(d.FragmentPath)) != configDir || !strings.HasPrefix(fragmentName, "lattice-linechain-") || filepath.Ext(fragmentName) != ".json" { + return fmt.Errorf("fragment path is outside the server-owned linechain config namespace") + } + wantSidecar := filepath.Join(filepath.Dir(configDir), "lattice-metadata.json") + if filepath.Clean(d.SidecarPath) != wantSidecar { + return fmt.Errorf("sidecar path must be %s", wantSidecar) + } + if err := validateParents(d.FragmentPath); err != nil { + return err + } + if err := validateParents(d.SidecarPath); err != nil { + return err + } + for _, want := range []string{d.PreviousFragmentSHA256, d.PreviousSidecarSHA256} { + if want != "" && !validSHA(want) { + return fmt.Errorf("previous artifact digest is invalid") + } + } + return nil +} + +func (m *Manager) checkRestartVerify(ctx context.Context, d Document) error { + bin := strings.TrimSpace(d.SingBoxBinary) + if bin == "" { + bin = "sing-box" + } + args := d.CheckArgs + if len(args) == 0 { + args = []string{"check", "-C", d.ConfigDir} + } + if out, err := m.run(ctx, bin, args...); err != nil { + return fmt.Errorf("sing-box check failed: %s", bounded(out)) + } + restart := d.RestartCommand + if len(restart) == 0 { + restart = []string{"systemctl", "restart", "sing-box"} + } + if out, err := m.run(ctx, restart[0], restart[1:]...); err != nil { + return fmt.Errorf("sing-box restart failed: %s", bounded(out)) + } + verify := d.VerifyCommand + if len(verify) == 0 { + verify = []string{"systemctl", "is-active", "--quiet", "sing-box"} + } + if out, err := m.run(ctx, verify[0], verify[1:]...); err != nil { + return fmt.Errorf("sing-box active verification failed: %s", bounded(out)) + } + return nil +} + +// ResolveAfterRun converts the host transaction into one stable terminal result. +func (m *Manager) ResolveAfterRun(ctx context.Context, task model.Task, result model.TaskResult) (model.TaskResult, error) { + path := m.journalPath(task.ID, task.LeaseID) + j, err := readJournal(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) && (result.ExitCode != 0 || result.Error != "") { + if result.FinishedAt.IsZero() { + result.FinishedAt = time.Now().UTC() + } + return result, nil + } + return result, err + } + if j.Result != nil { + return *j.Result, nil + } + if result.FinishedAt.IsZero() { + result.FinishedAt = time.Now().UTC() + } + if result.ExitCode == 0 && result.Error == "" && j.Phase == "desired_verified" { + if !pairMatches(j, true) { + return result, fmt.Errorf("desired linechain pair is not exact") + } + j.Phase = "terminal_desired" + } else { + if err := m.restoreOld(path, &j); err != nil { + return result, err + } + if result.ExitCode == 0 { + result.ExitCode = -1 + } + if result.Error == "" { + result.Error = "linechain helper did not leave a verified desired pair; exact old pair restored" + } + j.Phase = "terminal_old" + } + j.Result = &result + if err := writeJSON(path, j); err != nil { + return result, err + } + return result, nil +} + +// RequireRecovered resolves all non-terminal journals before network activity. +// complete is called only with a stable exact result; cleanup follows its +// confirmed durable completion. +func (m *Manager) RequireRecovered(ctx context.Context, complete func(model.TaskResult) error, nodeID string) error { + entries, err := os.ReadDir(m.dir) + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + if err := m.recoverOne(ctx, filepath.Join(m.dir, entry.Name()), complete, nodeID); err != nil { + return err + } + } + return nil +} + +func (m *Manager) recoverOne(ctx context.Context, path string, complete func(model.TaskResult) error, nodeID string) error { + j, err := readJournal(path) + if err != nil { + return err + } + if j.Result == nil { + result := model.TaskResult{TaskID: j.TaskID, LeaseID: j.LeaseID, NodeID: nodeID, ExitCode: -1, Error: "linechain transaction interrupted; exact old pair restored", FinishedAt: time.Now().UTC()} + if j.Phase == "desired_verified" && pairMatches(j, true) { + result.ExitCode = 0 + result.Error = "" + j.Phase = "terminal_desired" + } else { + if err := m.restoreOld(path, &j); err != nil { + return err + } + j.Phase = "terminal_old" + } + j.Result = &result + if err := writeJSON(path, j); err != nil { + return err + } + } + if complete != nil { + if err := complete(*j.Result); err != nil { + return err + } + return m.Cleanup(j.TaskID, j.LeaseID) + } + return nil +} + +func (m *Manager) Cleanup(taskID, leaseID string) error { + path := m.journalPath(taskID, leaseID) + for _, p := range []string{path + ".fragment.old", path + ".sidecar.old", path} { + if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return syncDir(m.dir) +} + +func (m *Manager) rollback(ctx context.Context, path string, j *journal, d Document, cause error) error { + if err := m.restoreOld(path, j); err != nil { + return fmt.Errorf("%v; rollback failed: %w", cause, err) + } + if err := m.checkRestartVerify(ctx, d); err != nil { + return fmt.Errorf("%v; exact old pair restored but service recovery failed: %s", cause, bounded([]byte(err.Error()))) + } + return cause +} + +func (m *Manager) restoreOld(path string, j *journal) error { + for _, a := range []struct { + dst, backup string + had bool + want string + }{{j.FragmentPath, path + ".fragment.old", j.FragmentHadOld, j.FragmentOld}, {j.SidecarPath, path + ".sidecar.old", j.SidecarHadOld, j.SidecarOld}} { + if a.had { + data, err := os.ReadFile(a.backup) + if err != nil { + return err + } + if digest(data) != a.want { + return fmt.Errorf("backup digest mismatch") + } + s := string(data) + if err := publish(a.dst, &s); err != nil { + return err + } + } else if err := publish(a.dst, nil); err != nil { + return err + } + } + if !pairMatches(*j, false) { + return fmt.Errorf("restored pair digest mismatch") + } + j.Phase = "old_restored" + return writeJSON(path, *j) +} + +func pairMatches(j journal, desired bool) bool { + a, ah, e1 := readCurrent(j.FragmentPath) + b, bh, e2 := readCurrent(j.SidecarPath) + if e1 != nil || e2 != nil { + return false + } + if desired { + return digestMaybe(a, ah) == j.FragmentNew && digestMaybe(b, bh) == j.SidecarNew + } + return ah == j.FragmentHadOld && bh == j.SidecarHadOld && digestMaybe(a, ah) == j.FragmentOld && digestMaybe(b, bh) == j.SidecarOld +} + +func readCurrent(path string) ([]byte, bool, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) { + return nil, false, fmt.Errorf("artifact is not a regular file: %s", path) + } + b, err := os.ReadFile(path) + return b, true, err +} +func requirePrevious(name string, data []byte, exists bool, want string) error { + if want == "" { + if exists { + return fmt.Errorf("unexpected existing %s artifact", name) + } + return nil + } + if !exists || digest(data) != strings.ToLower(want) { + return fmt.Errorf("%s artifact does not match previous digest", name) + } + return nil +} +func digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } +func digestPtr(s *string) string { + if s == nil { + return "" + } + return digest([]byte(*s)) +} +func digestMaybe(b []byte, exists bool) string { + if !exists { + return "" + } + return digest(b) +} +func validSHA(s string) bool { + b, e := hex.DecodeString(s) + return e == nil && len(b) == sha256.Size && s == strings.ToLower(s) +} +func bounded(b []byte) string { + b = bytes.TrimSpace(b) + if len(b) > maxDiagnostic { + b = b[:maxDiagnostic] + } + return string(b) +} +func (m *Manager) writeBackup(path string, data []byte, exists bool) error { + if !exists { + return nil + } + return writeFile(path, data) +} +func publish(path string, content *string) error { + if err := validateParents(path); err != nil { + return err + } + if content == nil { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDir(filepath.Dir(path)) + } + return writeFile(path, []byte(*content)) +} +func validateParents(path string) error { + dir := filepath.Dir(path) + info, err := os.Lstat(dir) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) { + return fmt.Errorf("artifact parent must be a real directory owned by the agent user") + } + return nil +} +func writeFile(path string, data []byte) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".lattice-linechain-*") + if err != nil { + return err + } + name := tmp.Name() + defer os.Remove(name) + if err = tmp.Chmod(0o600); err == nil { + _, err = tmp.Write(data) + } + if err == nil { + err = tmp.Sync() + } + if closeErr := tmp.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + if err = os.Rename(name, path); err != nil { + return err + } + f, err := os.Open(path) + if err == nil { + err = f.Sync() + _ = f.Close() + } + if err != nil { + return err + } + return syncDir(dir) +} +func writeJSON(path string, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + return writeFile(path, b) +} +func readJournal(path string) (journal, error) { + info, err := os.Lstat(path) + if err != nil { + return journal{}, err + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return journal{}, fmt.Errorf("invalid linechain journal") + } + b, err := os.ReadFile(path) + if err != nil { + return journal{}, err + } + var j journal + if err = json.Unmarshal(b, &j); err != nil { + return j, err + } + if j.Version != journalVersion || j.TaskID == "" || j.LeaseID == "" { + return j, fmt.Errorf("invalid linechain journal identity") + } + return j, nil +} +func syncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go new file mode 100644 index 0000000..690a602 --- /dev/null +++ b/internal/linechain/manager_test.go @@ -0,0 +1,210 @@ +package linechain + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/LatticeNet/lattice-sdk/model" +) + +func TestApplyCreateResolveAndCleanup(t *testing.T) { + m, dir := testManager(t) + configDir := filepath.Join(dir, "conf") + fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + mustMkdir(t, filepath.Dir(fragmentPath)) + fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}` + sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, + SingBoxBinary: "true", CheckArgs: []string{}, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + applyDoc(t, m, doc, "task-a", "lease-a") + result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()}) + if err != nil || result.ExitCode != 0 { + t.Fatalf("resolve: result=%+v err=%v", result, err) + } + journalBytes, err := os.ReadFile(m.journalPath("task-a", "lease-a")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(journalBytes), fragment) { + t.Fatal("journal contains credential-bearing desired bytes") + } + if err := m.Cleanup("task-a", "lease-a"); err != nil { + t.Fatal(err) + } +} + +func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { + m, dir := testManager(t) + mustMkdir(t, filepath.Join(dir, "conf")) + configDir := filepath.Join(dir, "conf") + fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err := os.WriteFile(fragmentPath, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + desired := "new" + doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &desired, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}, SingBoxBinary: "true"} + if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") { + t.Fatalf("unexpected error: %v", err) + } + if err := os.Remove(fragmentPath); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(dir, "target"), fragmentPath); err != nil { + t.Fatal(err) + } + if err := applyDocErr(m, doc, "task-b", "lease-b"); err == nil || !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCheckFailureRestoresExactOldPair(t *testing.T) { + m, dir := testManager(t) + mustMkdir(t, filepath.Join(dir, "conf")) + configDir := filepath.Join(dir, "conf") + fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + oldFragment, oldSidecar := "old-fragment", "old-sidecar" + for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} { + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + } + newFragment, newSidecar := "new-fragment", "new-sidecar" + doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, PreviousFragmentSHA256: digest([]byte(oldFragment)), PreviousSidecarSHA256: digest([]byte(oldSidecar)), Fragment: &newFragment, Sidecar: &newSidecar, SingBoxBinary: "false", RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") { + t.Fatalf("unexpected error: %v", err) + } + assertFile(t, fragmentPath, oldFragment) + assertFile(t, sidecarPath, oldSidecar) +} + +func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { + m, dir := testManager(t) + mustMkdir(t, filepath.Join(dir, "conf")) + fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-chain.json") + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + j := journal{Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, Phase: "prepared"} + path := m.journalPath(j.TaskID, j.LeaseID) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + var got model.TaskResult + if err := m.RequireRecovered(context.Background(), func(r model.TaskResult) error { got = r; return nil }, "node-a"); err != nil { + t.Fatal(err) + } + if got.ExitCode == 0 || got.TaskID != j.TaskID || got.LeaseID != j.LeaseID || got.FinishedAt.IsZero() { + t.Fatalf("bad recovered result: %+v", got) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("journal not cleaned: %v", err) + } +} + +func TestOpenIsExclusive(t *testing.T) { + dir := filepath.Join(t.TempDir(), "txn") + m, err := Open(dir) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if _, err := Open(dir); err == nil { + t.Fatal("second manager unexpectedly acquired lock") + } + if helper, err := OpenHelper(dir); err != nil { + t.Fatal(err) + } else { + _ = helper.Close() + } +} + +func TestOpenRecoversStaleLockFile(t *testing.T) { + dir := filepath.Join(t.TempDir(), "txn") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".lock"), []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + m, err := Open(dir) + if err != nil { + t.Fatalf("stale lock file blocked advisory lock: %v", err) + } + defer m.Close() +} + +func TestValidateDocumentBindsOwnedPaths(t *testing.T) { + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + fragment := "{}" + sidecar := "{}" + valid := Document{Version: 1, ConfigDir: configDir, FragmentPath: filepath.Join(configDir, "lattice-linechain-a.json"), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar} + if err := validateDocument(valid); err != nil { + t.Fatal(err) + } + invalid := valid + invalid.FragmentPath = filepath.Join(root, "outside.json") + if err := validateDocument(invalid); err == nil { + t.Fatal("outside fragment accepted") + } + invalid = valid + invalid.SidecarPath = filepath.Join(root, "other.json") + if err := validateDocument(invalid); err == nil { + t.Fatal("unowned sidecar accepted") + } +} + +func TestResolvePreflightFailureWithoutJournal(t *testing.T) { + m, _ := testManager(t) + finished := time.Now().UTC() + input := model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: -1, Error: "document rejected before host mutation", FinishedAt: finished} + got, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, input) + if err != nil { + t.Fatal(err) + } + if got != input { + t.Fatalf("preflight result changed: %+v", got) + } +} + +func testManager(t *testing.T) (*Manager, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "txn") + m, err := Open(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = m.Close() }) + return m, dir +} +func mustMkdir(t *testing.T, p string) { + t.Helper() + if err := os.MkdirAll(p, 0o700); err != nil { + t.Fatal(err) + } +} +func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) { + t.Helper() + if err := applyDocErr(m, d, task, lease); err != nil { + t.Fatal(err) + } +} +func applyDocErr(m *Manager, d Document, task, lease string) error { + b, _ := json.Marshal(d) + return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease) +} +func assertFile(t *testing.T, p, want string) { + t.Helper() + b, err := os.ReadFile(p) + if err != nil || string(b) != want { + t.Fatalf("file %s = %q err=%v", p, b, err) + } +} diff --git a/internal/taskexec/taskexec.go b/internal/taskexec/taskexec.go index 9fa7b0a..010f722 100644 --- a/internal/taskexec/taskexec.go +++ b/internal/taskexec/taskexec.go @@ -168,6 +168,9 @@ type Runner struct { // Trusted server-authored scripts use it for one-shot helper modes without // depending on PATH or inheriting the service process environment. AgentBinary string + // LinechainTxnDir is the private transaction root passed only to trusted + // server-authored helper invocations. + LinechainTxnDir string // getUID returns the effective uid of the agent process. It is a field so // tests can simulate "running as root" without actually being root. When // nil it defaults to os.Geteuid. @@ -393,6 +396,9 @@ func (r Runner) Run(task model.Task) model.TaskResult { if filepath.IsAbs(r.AgentBinary) { cmd.Env = append(cmd.Env, "LATTICE_AGENT_BIN="+r.AgentBinary) } + if filepath.IsAbs(r.LinechainTxnDir) { + cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TXN_DIR="+r.LinechainTxnDir) + } var stdout, stderr cappedBuffer stdout.limit = limit diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index 66d835a..1ba085b 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -194,6 +194,9 @@ func (s *Store) Complete(result model.TaskResult) (committed bool, err error) { if err != nil { return false, err } + if entry.State == stateDone && entry.Result != nil && reflect.DeepEqual(*entry.Result, result) { + return false, nil + } if entry.State != stateLeased { return false, fmt.Errorf("task lease %s is not awaiting completion", result.TaskID) } diff --git a/scripts/install.sh b/scripts/install.sh index d2400ec..84b2d75 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -17,6 +17,7 @@ token="${LATTICE_NODE_TOKEN:-}" bin_path="${LATTICE_AGENT_BIN:-$LATTICE_HOME/lattice-agent}" env_path="${LATTICE_AGENT_ENV:-$LATTICE_HOME/lattice-agent.env}" state_dir="${LATTICE_AGENT_STATE:-$LATTICE_HOME/state}" +linechain_txn_dir="${LATTICE_LINECHAIN_TXN_DIR:-$state_dir/linechain-txn}" service_name="${LATTICE_AGENT_SERVICE:-lattice-agent}" run_user="${LATTICE_AGENT_RUN_USER:-}" run_group="${LATTICE_AGENT_RUN_GROUP:-}" @@ -83,6 +84,7 @@ if [ "$(id -u)" -ne 0 ]; then LATTICE_TASK_CGROUP_CPU_MAX="${LATTICE_TASK_CGROUP_CPU_MAX:-}" \ LATTICE_TASK_WORK_ROOT="${LATTICE_TASK_WORK_ROOT:-}" \ LATTICE_TASK_OUTBOX_DIR="${LATTICE_TASK_OUTBOX_DIR:-}" \ + LATTICE_LINECHAIN_TXN_DIR="$linechain_txn_dir" \ LATTICE_AGENT_ALLOW_TERMINAL="${LATTICE_AGENT_ALLOW_TERMINAL:-}" \ LATTICE_TERMINAL_TRANSPORT="${LATTICE_TERMINAL_TRANSPORT:-}" \ LATTICE_IP_MODE="${LATTICE_IP_MODE:-}" LATTICE_IP_RESOLVERS="${LATTICE_IP_RESOLVERS:-}" \ @@ -196,7 +198,7 @@ load_existing_config() { LATTICE_AGENT_ALLOW_EXEC LATTICE_AGENT_ALLOW_ROOT_EXEC LATTICE_NO_EXEC \ LATTICE_TASK_CGROUP_ROOT LATTICE_TASK_CGROUP_MEMORY_MAX \ LATTICE_TASK_CGROUP_PIDS_MAX LATTICE_TASK_CGROUP_CPU_MAX \ - LATTICE_TASK_WORK_ROOT LATTICE_TASK_OUTBOX_DIR \ + LATTICE_TASK_WORK_ROOT LATTICE_TASK_OUTBOX_DIR LATTICE_LINECHAIN_TXN_DIR \ LATTICE_AGENT_ALLOW_TERMINAL LATTICE_TERMINAL_TRANSPORT LATTICE_IP_MODE \ LATTICE_IP_RESOLVERS LATTICE_IP_SCRIPT LATTICE_PUBLIC_IP LATTICE_PUBLIC_IP6 \ LATTICE_SSH_ALERTS LATTICE_SINGBOX_DISCOVER LATTICE_SINGBOX_BIN \ @@ -303,6 +305,23 @@ apply_service_identity_permissions() { chown "$run_user:$run_group" "$task_outbox_leaf" || die "cannot assign $task_outbox_leaf to $run_user:$run_group" chmod 0700 "$task_outbox_leaf" 2>/dev/null || true fi + chown "$run_user:$run_group" "$linechain_txn_dir" || die "cannot assign $linechain_txn_dir to $run_user:$run_group" + chmod 0700 "$linechain_txn_dir" 2>/dev/null || true +} + +prepare_linechain_txn_dir() { + case "$linechain_txn_dir" in + /|/bin|/boot|/dev|/etc|/lib|/lib32|/lib64|/proc|/root|/sbin|/sys|/usr|/var) + die "LATTICE_LINECHAIN_TXN_DIR must not be a filesystem or system root: $linechain_txn_dir" ;; + /*) ;; + *) die "LATTICE_LINECHAIN_TXN_DIR must be an absolute path" ;; + esac + [ ! -L "$linechain_txn_dir" ] || die "LATTICE_LINECHAIN_TXN_DIR must not be a symlink: $linechain_txn_dir" + if [ -e "$linechain_txn_dir" ] && [ ! -d "$linechain_txn_dir" ]; then + die "LATTICE_LINECHAIN_TXN_DIR must be a directory: $linechain_txn_dir" + fi + mkdir -p "$linechain_txn_dir" || die "cannot create LATTICE_LINECHAIN_TXN_DIR=$linechain_txn_dir" + chmod 0700 "$linechain_txn_dir" 2>/dev/null || true } prepare_task_work_root() { @@ -421,6 +440,7 @@ fi mkdir -p "$state_dir" prepare_task_work_root prepare_task_outbox_root +prepare_linechain_txn_dir chmod 0750 "$LATTICE_HOME" 2>/dev/null || true apply_service_identity_permissions @@ -475,6 +495,7 @@ LATTICE_TASK_CGROUP_PIDS_MAX=$(quote_env "${LATTICE_TASK_CGROUP_PIDS_MAX:-64}") LATTICE_TASK_CGROUP_CPU_MAX=$(quote_env "${LATTICE_TASK_CGROUP_CPU_MAX:-100000 100000}") LATTICE_TASK_WORK_ROOT=$(quote_env "${LATTICE_TASK_WORK_ROOT:-}") LATTICE_TASK_OUTBOX_DIR=$(quote_env "${LATTICE_TASK_OUTBOX_DIR:-}") +LATTICE_LINECHAIN_TXN_DIR=$(quote_env "$linechain_txn_dir") LATTICE_AGENT_ALLOW_TERMINAL=$(quote_env "${LATTICE_AGENT_ALLOW_TERMINAL:-0}") LATTICE_TERMINAL_TRANSPORT=$(quote_env "${LATTICE_TERMINAL_TRANSPORT:-poll}") LATTICE_IP_MODE=$(quote_env "${LATTICE_IP_MODE:-auto}") @@ -567,6 +588,7 @@ EOF LATTICE_TASK_CGROUP_CPU_MAX$(xml_escape "${LATTICE_TASK_CGROUP_CPU_MAX:-100000 100000}") LATTICE_TASK_WORK_ROOT$(xml_escape "${LATTICE_TASK_WORK_ROOT:-}") LATTICE_TASK_OUTBOX_DIR$(xml_escape "${LATTICE_TASK_OUTBOX_DIR:-}") + LATTICE_LINECHAIN_TXN_DIR$(xml_escape "$linechain_txn_dir") LATTICE_AGENT_ALLOW_TERMINAL$(xml_escape "${LATTICE_AGENT_ALLOW_TERMINAL:-0}") LATTICE_TERMINAL_TRANSPORT$(xml_escape "${LATTICE_TERMINAL_TRANSPORT:-poll}") LATTICE_IP_MODE$(xml_escape "${LATTICE_IP_MODE:-auto}") diff --git a/scripts/test-install-integrity.sh b/scripts/test-install-integrity.sh index 0ce1714..55d6727 100755 --- a/scripts/test-install-integrity.sh +++ b/scripts/test-install-integrity.sh @@ -121,7 +121,12 @@ for expected in \ 'chown "$run_user:$run_group" "$state_dir"' \ 'LATTICE_TASK_OUTBOX_DIR="${LATTICE_TASK_OUTBOX_DIR:-}"' \ 'LATTICE_TASK_OUTBOX_DIR=$(quote_env "${LATTICE_TASK_OUTBOX_DIR:-}")' \ - 'chown "$run_user:$run_group" "$task_outbox_leaf"' + 'chown "$run_user:$run_group" "$task_outbox_leaf"' \ + 'LATTICE_LINECHAIN_TXN_DIR="$linechain_txn_dir"' \ + 'LATTICE_LINECHAIN_TXN_DIR=$(quote_env "$linechain_txn_dir")' \ + 'prepare_linechain_txn_dir' \ + 'chmod 0700 "$linechain_txn_dir"' \ + 'chown "$run_user:$run_group" "$linechain_txn_dir"' do if ! grep -Fq "$expected" "$ROOT/scripts/install.sh"; then echo "installer non-root systemd contract missing: $expected" >&2 diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh new file mode 100755 index 0000000..9781ba7 --- /dev/null +++ b/scripts/test-linechain-e2e.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +bin="${LATTICE_SINGBOX_E2E_BIN:-}" +[ -n "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN is required" >&2; exit 1; } +case "$bin" in /*) ;; *) echo "LATTICE_SINGBOX_E2E_BIN must be absolute" >&2; exit 1;; esac +[ -x "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN must be executable" >&2; exit 1; } +"$bin" version >/dev/null 2>&1 || { echo "sing-box version failed" >&2; exit 1; } + +LATTICE_SINGBOX_E2E_BIN="$bin" go test ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From d47e092ede166eaa1ee996ceaf67a20b9eb76c6b Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:12:00 -0700 Subject: [PATCH 03/43] Harden node-agent chain authority and recovery handoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initial E3 lane left authority and crash-boundary gaps around lock files, runtime layout, result retention, and artifact binding. This follow-up refuses hostile lock state, resolves layout locally, retains only E3 results during cleanup retries, binds combined digests to raw artifact bytes, and makes transaction roots durable and private. Constraint: Server documents cannot choose arbitrary host paths Constraint: Generic task result behavior must remain unchanged Rejected: Repairing hostile lock modes in place | could mutate another owner’s state Confidence: high Scope-risk: moderate Directive: Keep cleanup retryable without retaining unrelated generic results Tested: full normal/race-cover/vet/gofmt/diff/install gates Not-tested: Full live server payload replay remains cross-lane integration work --- cmd/lattice-agent/linechain_e2e_test.go | 8 +- cmd/lattice-agent/main.go | 60 +++++- cmd/lattice-agent/main_test.go | 48 ++++- internal/linechain/lock_unix.go | 33 ++- internal/linechain/lock_unix_test.go | 83 ++++++++ internal/linechain/manager.go | 235 +++++++++++++++++----- internal/linechain/manager_test.go | 41 +++- internal/singboxdiscover/discover.go | 65 ++++++ internal/singboxdiscover/discover_test.go | 16 ++ internal/taskexec/taskexec.go | 7 +- 10 files changed, 517 insertions(+), 79 deletions(-) create mode 100644 internal/linechain/lock_unix_test.go diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index 7cb0513..9329b54 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -45,7 +45,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { sidecarPath := filepath.Join(root, "lattice-metadata.json") fragment := "{\n \"outbounds\": [{\"type\": \"direct\", \"tag\": \"linechain-direct\"}]\n}\n" sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}\n" - doc := linechain.Document{Version: 1, ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, SingBoxBinary: bin, CheckArgs: []string{"check", "-C", conf}, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + doc := linechain.BindDocument(linechain.Document{Version: 1, Operation: "create", ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) b, err := json.Marshal(doc) if err != nil { t.Fatal(err) @@ -55,6 +55,12 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { t.Fatal(err) } defer m.Close() + if err := m.ConfigureLayout(conf, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands(bin, []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } if err := m.Apply(context.Background(), bytes.NewReader(b), "e2e-task", "e2e-lease"); err != nil { t.Fatal(err) } diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index cb537e2..386e2ee 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -76,6 +76,13 @@ func reportedCapabilities() []string { return []string{durableTaskResultCapability, guardManagedSHACapability} } +func capabilitiesFor(linechainReady bool) []string { + if linechainReady { + return reportedCapabilities() + } + return []string{guardManagedSHACapability} +} + type agentConfig struct { Server string NodeID string @@ -157,6 +164,7 @@ type agentConfig struct { LogStateDir string TaskOutboxDir string LinechainTxnDir string + LinechainReady bool } type agentRuntimePayload struct { @@ -283,6 +291,9 @@ func main() { log.Fatalf("linechain transaction manager failed: %v", err) } defer manager.Close() + if err := manager.ConfigureLayout(os.Getenv("LATTICE_LINECHAIN_CONFIG_DIR"), os.Getenv("LATTICE_LINECHAIN_SIDECAR_PATH")); err != nil { + log.Fatalf("linechain local layout failed: %v", err) + } if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID")); err != nil { log.Fatalf("linechain apply failed: %v", err) } @@ -361,8 +372,24 @@ func main() { log.Fatalf("linechain transaction manager initialization failed: %v", err) } defer linechainManager.Close() - if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil { - log.Fatalf("linechain recovery blocked readiness: %v", err) + linechainConfigDir, linechainSidecarPath, layoutErr := singboxdiscover.ResolveRuntimeLayout(cfg.SingBoxMeta) + if layoutErr != nil { + log.Printf("warning: durable linechain tasks disabled: %v", layoutErr) + } else if err := linechainManager.ConfigureLayout(linechainConfigDir, linechainSidecarPath); err != nil { + log.Printf("warning: durable linechain tasks disabled: %v", err) + } else { + cfg.LinechainReady = true + } + for { + if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err == nil { + break + } else { + log.Printf("linechain recovery blocked readiness: %v", err) + if uploadErr := flushTaskResultsRetain(cfg, taskResults, true); uploadErr != nil { + log.Printf("linechain terminal result upload error: %v", uploadErr) + } + time.Sleep(cfg.Interval) + } } agentBinary, err := os.Executable() if err != nil { @@ -386,7 +413,7 @@ func main() { if err := postAgentJSON(cfg, "/api/agent/hello", map[string]any{ "version": version, "compatibility": compatibilityPayload(), - "capabilities": reportedCapabilities(), + "capabilities": capabilitiesFor(cfg.LinechainReady), "public_ip": cfg.PublicIP, "public_ipv6": cfg.PublicIPv6, "internal_ip": cfg.InternalIP, @@ -414,7 +441,7 @@ func main() { runner := taskexec.Runner{ AllowExec: cfg.AllowExec, AllowRoot: cfg.AllowRoot, Cgroup: cfg.taskCgroupConfig(), - WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary, LinechainTxnDir: linechainDir, + WorkdirRoot: cfg.TaskWorkRoot, AgentBinary: agentBinary, LinechainTxnDir: linechainDir, LinechainConfigDir: linechainConfigDir, LinechainSidecarPath: linechainSidecarPath, } monitors := newMonitorManager(cfg) logTailers := newLogTailManager(cfg) @@ -423,6 +450,9 @@ func main() { for { if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil { log.Printf("linechain recovery blocked cycle: %v", err) + if uploadErr := flushTaskResultsRetain(cfg, taskResults, true); uploadErr != nil { + log.Printf("linechain terminal result upload error: %v", uploadErr) + } <-ticker.C continue } @@ -715,7 +745,7 @@ func reportMetrics(cfg agentConfig) error { return postAgentJSON(cfg, "/api/agent/metrics", map[string]any{ "version": version, "compatibility": compatibilityPayload(), - "capabilities": reportedCapabilities(), + "capabilities": capabilitiesFor(cfg.LinechainReady), "agent_runtime": agentRuntimePayload{ AllowExec: cfg.AllowExec, AllowRootExec: cfg.AllowRoot, @@ -1163,6 +1193,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } if manager != nil { if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil { + _ = flushTaskResultsRetain(cfg, outbox, true) return err } } @@ -1177,7 +1208,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag return err } req.Header.Set("Authorization", "Bearer "+cfg.Token) - req.Header.Set(agentCapabilitiesHeader, strings.Join(reportedCapabilities(), ",")) + req.Header.Set(agentCapabilitiesHeader, strings.Join(capabilitiesFor(cfg.LinechainReady), ",")) resp, err := httpClient.Do(req) if err != nil { return err @@ -1258,7 +1289,10 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } if manager != nil && isLinechainTask(task) { if cleanupErr := manager.Cleanup(task.ID, task.LeaseID); cleanupErr != nil { - return fmt.Errorf("%v; cleanup confirmed linechain task: %w", journalErr, cleanupErr) + if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil { + return fmt.Errorf("%v; cleanup confirmed linechain task: %v; upload stable result: %w", journalErr, cleanupErr, uploadErr) + } + return fmt.Errorf("%v; cleanup confirmed linechain task: %w; result remains replayable", journalErr, cleanupErr) } } if flushErr := flushTaskResults(cfg, outbox); flushErr != nil { @@ -1269,7 +1303,10 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } if manager != nil && isLinechainTask(task) { if err := manager.Cleanup(task.ID, task.LeaseID); err != nil { - return fmt.Errorf("cleanup linechain task %s after durable outbox completion: %w", task.ID, err) + if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil { + return fmt.Errorf("cleanup linechain task %s: %v; upload stable result: %w", task.ID, err, uploadErr) + } + return fmt.Errorf("cleanup linechain task %s after durable outbox completion; result remains replayable: %w", task.ID, err) } } if err := flushTaskResults(cfg, outbox); err != nil { @@ -1299,6 +1336,10 @@ func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, } func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error { + return flushTaskResultsRetain(cfg, outbox, false) +} + +func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain bool) error { pending, err := outbox.Pending() if err != nil { return fmt.Errorf("read pending task results: %w", err) @@ -1312,6 +1353,9 @@ func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error { }, nil); err != nil { return fmt.Errorf("flush durable task result %s: %w", entry.Task.ID, err) } + if retain && isLinechainTask(entry.Task) { + continue + } if err := outbox.Remove(entry); err != nil { return fmt.Errorf("remove acknowledged task result %s: %w", entry.Task.ID, err) } diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index 639009b..d992a88 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -285,7 +285,7 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { return testResponse(http.StatusNotFound, ""), nil } })} - cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"} + cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret", LinechainReady: true} if err := runTasks(cfg, runner, store); err == nil { t.Fatal("first result upload should fail") @@ -310,14 +310,18 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { } type linechainTaskRunner struct { - manager *linechain.Manager - doc []byte - calls int + manager *linechain.Manager + doc []byte + calls int + afterApply func() } func (r *linechainTaskRunner) Run(task model.Task) model.TaskResult { r.calls++ err := r.manager.Apply(context.Background(), bytes.NewReader(r.doc), task.ID, task.LeaseID) + if err == nil && r.afterApply != nil { + r.afterApply() + } result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, StartedAt: time.Now().UTC(), FinishedAt: time.Now().UTC()} if err != nil { result.ExitCode = -1 @@ -345,18 +349,33 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { if err := os.Mkdir(conf, 0o700); err != nil { t.Fatal(err) } + if err := manager.ConfigureLayout(conf, filepath.Join(root, "lattice-metadata.json")); err != nil { + t.Fatal(err) + } + if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } fragmentPath := filepath.Join(conf, "lattice-linechain-chain.json") sidecarPath := filepath.Join(root, "lattice-metadata.json") fragment := "{\"outbounds\":[]}" sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}" - doc, err := json.Marshal(linechain.Document{Version: 1, ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, SingBoxBinary: "true", RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}}) + docValue := linechain.BindDocument(linechain.Document{Version: 1, Operation: "create", ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) + doc, err := json.Marshal(docValue) if err != nil { t.Fatal(err) } task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: `"$LATTICE_AGENT_BIN" -linechain-apply`, TimeoutSec: 10, OutputLimit: 1024} - runner := &linechainTaskRunner{manager: manager, doc: doc} + runner := &linechainTaskRunner{manager: manager, doc: doc, afterApply: func() { + manager.ConfigureCleanupForTest(func(path string) error { + if strings.HasSuffix(path, ".json") { + return errors.New("injected cleanup failure") + } + return os.Remove(path) + }, nil) + }} fetches := 0 posts := 0 + var posted []model.TaskResult httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { switch r.URL.Path { case "/api/agent/tasks": @@ -368,19 +387,30 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { return testResponse(http.StatusOK, "[]"), nil case "/api/agent/task-result": posts++ + var body struct { + Result model.TaskResult `json:"result"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + posted = append(posted, body.Result) return testResponse(http.StatusOK, `{"ok":true}`), nil default: return testResponse(http.StatusNotFound, ""), nil } })} cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"} - if err := runTasks(cfg, runner, outbox, manager); err != nil { - t.Fatal(err) + if err := runTasks(cfg, runner, outbox, manager); err == nil { + t.Fatal("cleanup failure should remain readiness-visible") } + if pending, err := outbox.Pending(); err != nil || len(pending) != 1 { + t.Fatalf("completed result not retained: %+v %v", pending, err) + } + manager.ConfigureCleanupForTest(nil, nil) if err := runTasks(cfg, runner, outbox, manager); err != nil { t.Fatal(err) } - if runner.calls != 1 || posts != 1 { + if runner.calls != 1 || posts != 2 || len(posted) != 2 || !reflect.DeepEqual(posted[0], posted[1]) { t.Fatalf("runner calls=%d posts=%d", runner.calls, posts) } if entries, err := os.ReadDir(txnDir); err != nil { diff --git a/internal/linechain/lock_unix.go b/internal/linechain/lock_unix.go index 43dd7d3..cf63d4c 100644 --- a/internal/linechain/lock_unix.go +++ b/internal/linechain/lock_unix.go @@ -14,18 +14,43 @@ func lockManager(path string) (*os.File, error) { return nil, err } f := os.NewFile(uintptr(fd), path) - if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if f == nil { + _ = syscall.Close(fd) + return nil, fmt.Errorf("open linechain manager lock: invalid file descriptor") + } + info, err := f.Stat() + if err != nil { _ = f.Close() - return nil, fmt.Errorf("linechain transaction manager already open: %w", err) + return nil, fmt.Errorf("inspect linechain manager lock: %w", err) } - if err := f.Chmod(0o600); err != nil { - _ = syscall.Flock(fd, syscall.LOCK_UN) + if err := validateLockInfo(info, os.Geteuid()); err != nil { _ = f.Close() return nil, err } + if err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + return nil, fmt.Errorf("linechain transaction manager already open: %w", err) + } return f, nil } +func validateLockInfo(info os.FileInfo, effectiveUID int) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("inspect linechain manager lock ownership: unsupported stat data") + } + if !info.Mode().IsRegular() { + return fmt.Errorf("linechain manager lock must be a regular file") + } + if stat.Uid != uint32(effectiveUID) { + return fmt.Errorf("linechain manager lock must be owned by effective user %d", effectiveUID) + } + if info.Mode().Perm() != 0o600 { + return fmt.Errorf("linechain manager lock permissions are %o, want 600", info.Mode().Perm()) + } + return nil +} + func unlockManager(f *os.File) error { err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN) if closeErr := f.Close(); err == nil { diff --git a/internal/linechain/lock_unix_test.go b/internal/linechain/lock_unix_test.go new file mode 100644 index 0000000..7909fb6 --- /dev/null +++ b/internal/linechain/lock_unix_test.go @@ -0,0 +1,83 @@ +//go:build linux || darwin || freebsd + +package linechain + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestOpenRefusesSymlinkLockWithoutChangingTarget(t *testing.T) { + dir := filepath.Join(t.TempDir(), "txn") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + victim := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(victim, []byte("unchanged"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, filepath.Join(dir, ".lock")); err != nil { + t.Fatal(err) + } + if m, err := Open(dir); err == nil { + _ = m.Close() + t.Fatal("symlink lock accepted") + } + b, err := os.ReadFile(victim) + if err != nil || string(b) != "unchanged" { + t.Fatalf("victim changed: %q %v", b, err) + } +} + +func TestOpenRefusesFIFOAndInsecureLockMode(t *testing.T) { + for _, tc := range []struct { + name string + make func(string) error + }{ + {"fifo", func(path string) error { return syscall.Mkfifo(path, 0o600) }}, + {"mode", func(path string) error { return os.WriteFile(path, nil, 0o644) }}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "txn") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, ".lock") + if err := tc.make(path); err != nil { + t.Fatal(err) + } + if tc.name == "mode" { + _ = os.Chmod(path, 0o644) + } + if m, err := Open(dir); err == nil { + _ = m.Close() + t.Fatal("hostile lock accepted") + } + }) + } +} + +type lockInfoWithStat struct { + os.FileInfo + stat syscall.Stat_t +} + +func (i lockInfoWithStat) Sys() any { return &i.stat } + +func TestLockValidationRefusesWrongOwner(t *testing.T) { + path := filepath.Join(t.TempDir(), "lock") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + stat := *(info.Sys().(*syscall.Stat_t)) + stat.Uid = uint32(os.Geteuid() + 1) + if err := validateLockInfo(lockInfoWithStat{FileInfo: info, stat: stat}, os.Geteuid()); err == nil { + t.Fatal("wrong-owner lock accepted") + } +} diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index e69eaec..1f800fc 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -29,18 +29,34 @@ const ( // Document is the bounded server-rendered input consumed by -linechain-apply. // Desired content is never copied into the journal; only its digest is stored. type Document struct { - Version int `json:"version"` - ConfigDir string `json:"config_dir"` - FragmentPath string `json:"fragment_path"` - SidecarPath string `json:"sidecar_path"` - PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` - PreviousSidecarSHA256 string `json:"previous_sidecar_sha256,omitempty"` - Fragment *string `json:"fragment,omitempty"` - Sidecar *string `json:"sidecar,omitempty"` - SingBoxBinary string `json:"singbox_binary,omitempty"` - CheckArgs []string `json:"check_args,omitempty"` - RestartCommand []string `json:"restart_command,omitempty"` - VerifyCommand []string `json:"verify_command,omitempty"` + Version int `json:"version"` + Operation string `json:"operation"` + ConfigDir string `json:"config_dir"` + FragmentPath string `json:"fragment_path"` + SidecarPath string `json:"sidecar_path"` + PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` + PreviousSidecarSHA256 string `json:"previous_sidecar_sha256,omitempty"` + Fragment *string `json:"fragment,omitempty"` + Sidecar *string `json:"sidecar,omitempty"` + FragmentSHA256 string `json:"fragment_sha256,omitempty"` + SidecarSHA256 string `json:"sidecar_sha256,omitempty"` + CombinedSHA256 string `json:"combined_sha256"` +} + +// BindDocument computes the deterministic desired artifact binding. +func BindDocument(d Document) Document { + d.FragmentSHA256 = digestPtr(d.Fragment) + d.SidecarSHA256 = digestPtr(d.Sidecar) + var pair []byte + if d.Fragment != nil { + pair = append(pair, []byte(*d.Fragment)...) + } + pair = append(pair, 0) + if d.Sidecar != nil { + pair = append(pair, []byte(*d.Sidecar)...) + } + d.CombinedSHA256 = digest(pair) + return d } type journal struct { @@ -60,9 +76,58 @@ type journal struct { } type Manager struct { - dir string - lock *os.File - run func(context.Context, string, ...string) ([]byte, error) + dir string + lock *os.File + run func(context.Context, string, ...string) ([]byte, error) + configDir string + sidecarPath string + remove func(string) error + syncDirectory func(string) error + singBoxBinary string + restartCommand []string + verifyCommand []string +} + +func (m *Manager) ConfigureLayout(configDir, sidecarPath string) error { + configDir = filepath.Clean(strings.TrimSpace(configDir)) + sidecarPath = filepath.Clean(strings.TrimSpace(sidecarPath)) + if !filepath.IsAbs(configDir) || !filepath.IsAbs(sidecarPath) { + return fmt.Errorf("linechain layout paths must be absolute") + } + if err := validateParents(filepath.Join(configDir, "placeholder")); err != nil { + return err + } + if err := validateParents(sidecarPath); err != nil { + return err + } + m.configDir = configDir + m.sidecarPath = sidecarPath + return nil +} +func (m *Manager) Configured() bool { return m != nil && m.configDir != "" && m.sidecarPath != "" } + +// ConfigureCommands is an integration-test seam. Production uses fixed local +// command vectors; task documents cannot select executables or arguments. +func (m *Manager) ConfigureCommands(binary string, restart, verify []string) error { + if strings.TrimSpace(binary) == "" || len(restart) == 0 || len(verify) == 0 { + return fmt.Errorf("linechain command vectors must be non-empty") + } + m.singBoxBinary = binary + m.restartCommand = append([]string(nil), restart...) + m.verifyCommand = append([]string(nil), verify...) + return nil +} + +// ConfigureCleanupForTest injects cleanup fault seams for crash-boundary tests. +func (m *Manager) ConfigureCleanupForTest(remove func(string) error, syncDirectory func(string) error) { + if remove == nil { + remove = os.Remove + } + if syncDirectory == nil { + syncDirectory = syncDir + } + m.remove = remove + m.syncDirectory = syncDirectory } // Open takes exclusive ownership of a private absolute transaction directory. @@ -76,7 +141,12 @@ func Open(dir string) (*Manager, error) { return nil, fmt.Errorf("linechain transaction manager already open: %w", err) } m.lock = lock - return m, syncDir(m.dir) + if err := syncDir(m.dir); err != nil { + _ = unlockManager(lock) + m.lock = nil + return nil, err + } + return m, nil } // OpenHelper opens the transaction directory without taking the manager lock. @@ -88,23 +158,67 @@ func open(dir string) (*Manager, error) { if dir == "." || !filepath.IsAbs(dir) || dir == string(filepath.Separator) { return nil, fmt.Errorf("linechain transaction directory must be absolute and non-root") } - if err := os.MkdirAll(dir, 0o700); err != nil { + if err := makeDurablePrivateDir(dir, syncDir); err != nil { return nil, err } info, err := os.Lstat(dir) if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { return nil, fmt.Errorf("linechain transaction path must be a real directory") } - if err := os.Chmod(dir, 0o700); err != nil { - return nil, err + if !ownedPath(info) || info.Mode().Perm() != 0o700 { + return nil, fmt.Errorf("linechain transaction directory must be agent-owned with exact mode 0700") } m := &Manager{dir: dir} + m.remove = os.Remove + m.syncDirectory = syncDir + m.singBoxBinary = "sing-box" + m.restartCommand = []string{"systemctl", "restart", "sing-box"} + m.verifyCommand = []string{"systemctl", "is-active", "--quiet", "sing-box"} m.run = func(ctx context.Context, name string, args ...string) ([]byte, error) { return exec.CommandContext(ctx, name, args...).CombinedOutput() } return m, nil } +func makeDurablePrivateDir(dir string, syncDirectory func(string) error) error { + dir = filepath.Clean(dir) + missing := []string{} + current := dir + for { + info, err := os.Lstat(current) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("path component must be a real directory: %s", current) + } + break + } + if !errors.Is(err, os.ErrNotExist) { + return err + } + missing = append(missing, current) + parent := filepath.Dir(current) + if parent == current { + return fmt.Errorf("no existing parent for linechain transaction directory") + } + current = parent + } + if parent := filepath.Dir(current); parent != current { + if err := syncDirectory(parent); err != nil { + return fmt.Errorf("confirm existing directory %s: %w", current, err) + } + } + for i := len(missing) - 1; i >= 0; i-- { + path := missing[i] + if err := os.Mkdir(path, 0o700); err != nil { + return err + } + if err := syncDirectory(filepath.Dir(path)); err != nil { + return fmt.Errorf("sync parent after creating %s: %w", path, err) + } + } + return nil +} + func (m *Manager) Close() error { if m == nil || m.lock == nil { return nil @@ -131,7 +245,7 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := dec.Decode(&d); err != nil { return fmt.Errorf("decode linechain document: %w", err) } - if err := validateDocument(d); err != nil { + if err := m.validateDocument(d); err != nil { return err } path := m.journalPath(taskID, leaseID) @@ -186,11 +300,37 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string return writeJSON(path, j) } -func validateDocument(d Document) error { +func (m *Manager) validateDocument(d Document) error { if d.Version != journalVersion { return fmt.Errorf("unsupported linechain document version %d", d.Version) } + switch d.Operation { + case "create": + if d.PreviousFragmentSHA256 != "" || d.PreviousSidecarSHA256 != "" || d.Fragment == nil || d.Sidecar == nil { + return fmt.Errorf("create document has inconsistent old/desired shape") + } + case "replace": + if d.PreviousFragmentSHA256 == "" || d.PreviousSidecarSHA256 == "" || d.Fragment == nil || d.Sidecar == nil { + return fmt.Errorf("replace document has inconsistent old/desired shape") + } + case "remove": + if d.PreviousFragmentSHA256 == "" || d.PreviousSidecarSHA256 == "" || d.Fragment != nil || d.Sidecar == nil { + return fmt.Errorf("remove document has inconsistent old/desired shape") + } + default: + return fmt.Errorf("unsupported linechain operation %q", d.Operation) + } + bound := BindDocument(d) + if d.FragmentSHA256 != bound.FragmentSHA256 || d.SidecarSHA256 != bound.SidecarSHA256 || d.CombinedSHA256 != bound.CombinedSHA256 { + return fmt.Errorf("linechain desired artifact digest binding mismatch") + } configDir := filepath.Clean(d.ConfigDir) + if !m.Configured() { + return fmt.Errorf("linechain runtime layout is unresolved") + } + if configDir != m.configDir { + return fmt.Errorf("config_dir does not match the locally resolved sing-box directory") + } if !filepath.IsAbs(configDir) || configDir == string(filepath.Separator) { return fmt.Errorf("config_dir must be absolute and non-root") } @@ -209,9 +349,9 @@ func validateDocument(d Document) error { if filepath.Dir(filepath.Clean(d.FragmentPath)) != configDir || !strings.HasPrefix(fragmentName, "lattice-linechain-") || filepath.Ext(fragmentName) != ".json" { return fmt.Errorf("fragment path is outside the server-owned linechain config namespace") } - wantSidecar := filepath.Join(filepath.Dir(configDir), "lattice-metadata.json") + wantSidecar := m.sidecarPath if filepath.Clean(d.SidecarPath) != wantSidecar { - return fmt.Errorf("sidecar path must be %s", wantSidecar) + return fmt.Errorf("sidecar path must match locally resolved %s", wantSidecar) } if err := validateParents(d.FragmentPath); err != nil { return err @@ -228,28 +368,14 @@ func validateDocument(d Document) error { } func (m *Manager) checkRestartVerify(ctx context.Context, d Document) error { - bin := strings.TrimSpace(d.SingBoxBinary) - if bin == "" { - bin = "sing-box" - } - args := d.CheckArgs - if len(args) == 0 { - args = []string{"check", "-C", d.ConfigDir} - } - if out, err := m.run(ctx, bin, args...); err != nil { + if out, err := m.run(ctx, m.singBoxBinary, "check", "-C", m.configDir); err != nil { return fmt.Errorf("sing-box check failed: %s", bounded(out)) } - restart := d.RestartCommand - if len(restart) == 0 { - restart = []string{"systemctl", "restart", "sing-box"} - } + restart := m.restartCommand if out, err := m.run(ctx, restart[0], restart[1:]...); err != nil { return fmt.Errorf("sing-box restart failed: %s", bounded(out)) } - verify := d.VerifyCommand - if len(verify) == 0 { - verify = []string{"systemctl", "is-active", "--quiet", "sing-box"} - } + verify := m.verifyCommand if out, err := m.run(ctx, verify[0], verify[1:]...); err != nil { return fmt.Errorf("sing-box active verification failed: %s", bounded(out)) } @@ -333,6 +459,9 @@ func (m *Manager) recoverOne(ctx context.Context, path string, complete func(mod if err := m.restoreOld(path, &j); err != nil { return err } + if err := m.checkRestartVerify(ctx, Document{}); err != nil { + return fmt.Errorf("recover old linechain pair service state: %w", err) + } j.Phase = "terminal_old" } j.Result = &result @@ -352,11 +481,11 @@ func (m *Manager) recoverOne(ctx context.Context, path string, complete func(mod func (m *Manager) Cleanup(taskID, leaseID string) error { path := m.journalPath(taskID, leaseID) for _, p := range []string{path + ".fragment.old", path + ".sidecar.old", path} { - if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { + if err := m.remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } - return syncDir(m.dir) + return m.syncDirectory(m.dir) } func (m *Manager) rollback(ctx context.Context, path string, j *journal, d Document, cause error) error { @@ -480,12 +609,22 @@ func publish(path string, content *string) error { } func validateParents(path string) error { dir := filepath.Dir(path) - info, err := os.Lstat(dir) - if err != nil { - return err - } - if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) { - return fmt.Errorf("artifact parent must be a real directory owned by the agent user") + for { + info, err := os.Lstat(dir) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("artifact parent must be a real directory") + } + if dir == filepath.Dir(path) && !ownedPath(info) { + return fmt.Errorf("artifact parent must be owned by the agent user") + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent } return nil } diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index 690a602..26d3d73 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -19,10 +19,15 @@ func TestApplyCreateResolveAndCleanup(t *testing.T) { fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") mustMkdir(t, filepath.Dir(fragmentPath)) + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}` sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar, - SingBoxBinary: "true", CheckArgs: []string{}, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + doc := Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar} applyDoc(t, m, doc, "task-a", "lease-a") result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()}) if err != nil || result.ExitCode != 0 { @@ -46,11 +51,18 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { configDir := filepath.Join(dir, "conf") fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } if err := os.WriteFile(fragmentPath, []byte("old"), 0o600); err != nil { t.Fatal(err) } desired := "new" - doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &desired, RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}, SingBoxBinary: "true"} + desiredSidecar := "{}" + doc := Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &desired, Sidecar: &desiredSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") { t.Fatalf("unexpected error: %v", err) } @@ -71,6 +83,12 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { configDir := filepath.Join(dir, "conf") fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("false", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } oldFragment, oldSidecar := "old-fragment", "old-sidecar" for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} { if err := os.WriteFile(path, []byte(data), 0o600); err != nil { @@ -78,7 +96,7 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { } } newFragment, newSidecar := "new-fragment", "new-sidecar" - doc := Document{Version: 1, ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, PreviousFragmentSHA256: digest([]byte(oldFragment)), PreviousSidecarSHA256: digest([]byte(oldSidecar)), Fragment: &newFragment, Sidecar: &newSidecar, SingBoxBinary: "false", RestartCommand: []string{"true"}, VerifyCommand: []string{"true"}} + doc := Document{Version: 1, Operation: "replace", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, PreviousFragmentSHA256: digest([]byte(oldFragment)), PreviousSidecarSHA256: digest([]byte(oldSidecar)), Fragment: &newFragment, Sidecar: &newSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") { t.Fatalf("unexpected error: %v", err) } @@ -91,6 +109,8 @@ func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { mustMkdir(t, filepath.Join(dir, "conf")) fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-chain.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err:=m.ConfigureLayout(filepath.Join(dir,"conf"),sidecarPath);err!=nil{t.Fatal(err)} + if err:=m.ConfigureCommands("true",[]string{"true"},[]string{"true"});err!=nil{t.Fatal(err)} j := journal{Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, Phase: "prepared"} path := m.journalPath(j.TaskID, j.LeaseID) if err := writeJSON(path, j); err != nil { @@ -141,23 +161,27 @@ func TestOpenRecoversStaleLockFile(t *testing.T) { } func TestValidateDocumentBindsOwnedPaths(t *testing.T) { + m, _ := testManager(t) root := t.TempDir() configDir := filepath.Join(root, "conf") mustMkdir(t, configDir) fragment := "{}" sidecar := "{}" - valid := Document{Version: 1, ConfigDir: configDir, FragmentPath: filepath.Join(configDir, "lattice-linechain-a.json"), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar} - if err := validateDocument(valid); err != nil { + if err := m.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil { + t.Fatal(err) + } + valid := BindDocument(Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: filepath.Join(configDir, "lattice-linechain-a.json"), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar}) + if err := m.validateDocument(valid); err != nil { t.Fatal(err) } invalid := valid invalid.FragmentPath = filepath.Join(root, "outside.json") - if err := validateDocument(invalid); err == nil { + if err := m.validateDocument(invalid); err == nil { t.Fatal("outside fragment accepted") } invalid = valid invalid.SidecarPath = filepath.Join(root, "other.json") - if err := validateDocument(invalid); err == nil { + if err := m.validateDocument(invalid); err == nil { t.Fatal("unowned sidecar accepted") } } @@ -198,6 +222,7 @@ func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) { } } func applyDocErr(m *Manager, d Document, task, lease string) error { + d = BindDocument(d) b, _ := json.Marshal(d) return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease) } diff --git a/internal/singboxdiscover/discover.go b/internal/singboxdiscover/discover.go index 309d9ac..32b2531 100644 --- a/internal/singboxdiscover/discover.go +++ b/internal/singboxdiscover/discover.go @@ -175,6 +175,14 @@ func discoverRuntimeConfig(source Source, nodeID string, at time.Time) (model.Si return inv, nil } +// DiscoverRuntimeFiles parses an explicit locally resolved runtime config set. +// It is used by recovery/E2E verification after the same bounded layout resolver +// has established file authority. +func DiscoverRuntimeFiles(nodeID string, files []string, metaPath string) (model.SingBoxInventory, error) { + copyFiles:=append([]string(nil),files...) + return discoverRuntimeConfig(Source{MetaPath:metaPath,runtimeFiles:func()[]string{return copyFiles}},nodeID,time.Now().UTC()) +} + // loadSingBoxRuntimeConfigFiles locates and reads the on-box sing-box config // files (the running process's -c/-C paths plus the /etc/sing-box defaults) and // returns each one that parsed successfully. Returns an empty slice when none @@ -696,6 +704,63 @@ func singBoxProcessArgs() [][]string { return out } +// ResolveRuntimeLayout returns the one locally observed sing-box -C directory +// and the design-17 sidecar path. It never trusts a server task document to +// choose writable host paths. +func ResolveRuntimeLayout(metaPath string) (string, string, error) { + return resolveRuntimeLayout(singBoxProcessArgs(), metaPath) +} + +func resolveRuntimeLayout(processes [][]string, metaPath string) (string, string, error) { + dirs := map[string]struct{}{} + for _, args := range processes { + for i := 0; i < len(args); i++ { + arg := args[i] + var value string + switch arg { + case "-C", "--config-directory": + if i+1 < len(args) { + i++ + value = args[i] + } + default: + for _, prefix := range []string{"-C=", "--config-directory="} { + if v, ok := strings.CutPrefix(arg, prefix); ok { + value = v + } + } + } + if value != "" && filepath.IsAbs(value) { + dirs[filepath.Clean(value)] = struct{}{} + } + } + } + if len(dirs) == 0 { + if info, err := os.Lstat("/etc/sing-box/conf"); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + dirs["/etc/sing-box/conf"] = struct{}{} + } + } + if len(dirs) != 1 { + return "", "", fmt.Errorf("resolve sing-box config directory: found %d active -C directories", len(dirs)) + } + var configDir string + for dir := range dirs { + configDir = dir + } + info, err := os.Lstat(configDir) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return "", "", fmt.Errorf("resolve sing-box config directory: path is not a real directory") + } + metaPath = strings.TrimSpace(metaPath) + if metaPath == "" { + metaPath = defaultMetaPath + } + if !filepath.IsAbs(metaPath) { + return "", "", fmt.Errorf("resolve sing-box sidecar: path must be absolute") + } + return configDir, filepath.Clean(metaPath), nil +} + func containsArg(args []string, want string) bool { for _, arg := range args { if arg == want { diff --git a/internal/singboxdiscover/discover_test.go b/internal/singboxdiscover/discover_test.go index 5dab8e4..015c7dc 100644 --- a/internal/singboxdiscover/discover_test.go +++ b/internal/singboxdiscover/discover_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -50,6 +51,21 @@ func TestDiscoverParsesListAndVersion(t *testing.T) { } } +func TestResolveRuntimeLayoutUsesLocalProcessAuthority(t *testing.T) { + dir := t.TempDir() + meta := filepath.Join(t.TempDir(), "lattice-metadata.json") + config, sidecar, err := resolveRuntimeLayout([][]string{{"/usr/bin/sing-box", "run", "-C", dir}}, meta) + if err != nil { + t.Fatal(err) + } + if config != dir || sidecar != meta { + t.Fatalf("layout = %q %q", config, sidecar) + } + if _, _, err := resolveRuntimeLayout([][]string{{"sing-box", "run", "-C", dir}, {"sing-box", "run", "-C", t.TempDir()}}, meta); err == nil { + t.Fatal("ambiguous config directories accepted") + } +} + func TestDiscoverListFailureReportsErrorStatus(t *testing.T) { src := Source{ runner: func(_ context.Context, _ string, args ...string) ([]byte, error) { diff --git a/internal/taskexec/taskexec.go b/internal/taskexec/taskexec.go index 010f722..84d052d 100644 --- a/internal/taskexec/taskexec.go +++ b/internal/taskexec/taskexec.go @@ -170,7 +170,9 @@ type Runner struct { AgentBinary string // LinechainTxnDir is the private transaction root passed only to trusted // server-authored helper invocations. - LinechainTxnDir string + LinechainTxnDir string + LinechainConfigDir string + LinechainSidecarPath string // getUID returns the effective uid of the agent process. It is a field so // tests can simulate "running as root" without actually being root. When // nil it defaults to os.Geteuid. @@ -399,6 +401,9 @@ func (r Runner) Run(task model.Task) model.TaskResult { if filepath.IsAbs(r.LinechainTxnDir) { cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TXN_DIR="+r.LinechainTxnDir) } + if filepath.IsAbs(r.LinechainConfigDir) && filepath.IsAbs(r.LinechainSidecarPath) { + cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_CONFIG_DIR="+r.LinechainConfigDir, "LATTICE_LINECHAIN_SIDECAR_PATH="+r.LinechainSidecarPath) + } var stdout, stderr cappedBuffer stdout.limit = limit From 25e8c20570a033ed2ca40067d71a7ea61b5609ab Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:12:50 -0700 Subject: [PATCH 04/43] Keep authority hardening formatting stable The follow-up touched existing tests and runtime layout discovery while closing the crash and authority gaps. Normalize those files so CI applies the same formatting gate as local verification. Constraint: CI rejects any gofmt drift Confidence: high Scope-risk: narrow Tested: gofmt and targeted Go tests Not-tested: none --- internal/linechain/manager_test.go | 8 ++++++-- internal/singboxdiscover/discover.go | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index 26d3d73..5f70be4 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -109,8 +109,12 @@ func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { mustMkdir(t, filepath.Join(dir, "conf")) fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-chain.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") - if err:=m.ConfigureLayout(filepath.Join(dir,"conf"),sidecarPath);err!=nil{t.Fatal(err)} - if err:=m.ConfigureCommands("true",[]string{"true"},[]string{"true"});err!=nil{t.Fatal(err)} + if err := m.ConfigureLayout(filepath.Join(dir, "conf"), sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } j := journal{Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, Phase: "prepared"} path := m.journalPath(j.TaskID, j.LeaseID) if err := writeJSON(path, j); err != nil { diff --git a/internal/singboxdiscover/discover.go b/internal/singboxdiscover/discover.go index 32b2531..93a9ad4 100644 --- a/internal/singboxdiscover/discover.go +++ b/internal/singboxdiscover/discover.go @@ -179,8 +179,8 @@ func discoverRuntimeConfig(source Source, nodeID string, at time.Time) (model.Si // It is used by recovery/E2E verification after the same bounded layout resolver // has established file authority. func DiscoverRuntimeFiles(nodeID string, files []string, metaPath string) (model.SingBoxInventory, error) { - copyFiles:=append([]string(nil),files...) - return discoverRuntimeConfig(Source{MetaPath:metaPath,runtimeFiles:func()[]string{return copyFiles}},nodeID,time.Now().UTC()) + copyFiles := append([]string(nil), files...) + return discoverRuntimeConfig(Source{MetaPath: metaPath, runtimeFiles: func() []string { return copyFiles }}, nodeID, time.Now().UTC()) } // loadSingBoxRuntimeConfigFiles locates and reads the on-box sing-box config From 141cf19152a6f976419baf9a8567f9acc3242133 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:16:36 -0700 Subject: [PATCH 05/43] Require explicit E3 task protocol markers Generic durable shell tasks must not be classified as replayable linechain operations merely because their script mentions a helper string. Require the server-authored E3 marker, helper invocation, and transaction-root binding together. Constraint: Generic result endpoints are not replay-safe Confidence: high Scope-risk: narrow Directive: Server task rendering must emit lattice-linechain-e3-v1 exactly Tested: go test, race-cover, vet, gofmt, diff Not-tested: full live server payload replay --- cmd/lattice-agent/main.go | 3 ++- cmd/lattice-agent/main_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 386e2ee..49af585 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1317,7 +1317,8 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } func isLinechainTask(task model.Task) bool { - return strings.Contains(task.Script, "-linechain-apply") + const marker = "# lattice-linechain-e3-v1\n" + return strings.HasPrefix(task.Script, marker) && strings.Contains(task.Script, "-linechain-apply") && strings.Contains(task.Script, "LATTICE_LINECHAIN_TXN_DIR") } func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index d992a88..98fc36c 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -364,7 +364,7 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { if err != nil { t.Fatal(err) } - task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: `"$LATTICE_AGENT_BIN" -linechain-apply`, TimeoutSec: 10, OutputLimit: 1024} + task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: "# lattice-linechain-e3-v1\n\"$LATTICE_LINECHAIN_TXN_DIR\"=1 \"$LATTICE_AGENT_BIN\" -linechain-apply", TimeoutSec: 10, OutputLimit: 1024} runner := &linechainTaskRunner{manager: manager, doc: doc, afterApply: func() { manager.ConfigureCleanupForTest(func(path string) error { if strings.HasSuffix(path, ".json") { From 6b9383c5c88574f5ebb6f43dece08bd1062106c5 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:24:12 -0700 Subject: [PATCH 06/43] Bind durable linechain tasks to local artifact layout Use explicit leased protocol metadata for live classification and derive artifact paths from the agent-resolved sing-box layout while retaining a narrow legacy recovery marker. Constraint: Existing pending journals do not carry leased protocol metadata Rejected: Trust server-supplied absolute paths | permits host redirection Confidence: medium Scope-risk: moderate Directive: Remove legacy marker fallback once all persisted outbox entries carry durable_protocol Tested: go test ./internal/linechain ./cmd/lattice-agent Not-tested: authentic external sing-box E2E harness --- cmd/lattice-agent/main.go | 29 +++++++++++++++++++++-------- internal/linechain/manager.go | 19 ++++++++++++++++++- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 49af585..13b9ce9 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1183,7 +1183,8 @@ type taskResultOutbox interface { type leasedAgentTask struct { model.Task - DurableResult bool `json:"durable_result"` + DurableResult bool `json:"durable_result"` + DurableProtocol string `json:"durable_protocol,omitempty"` } func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, managers ...*linechain.Manager) error { @@ -1269,7 +1270,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag debugf(cfg, "task start: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec) result := runner.Run(task) result.NodeID = cfg.NodeID - if manager != nil && isLinechainTask(task) { + if manager != nil && isLinechainTask(leased) { result, err = manager.ResolveAfterRun(context.Background(), task, result) if err != nil { return fmt.Errorf("resolve linechain task %s: %w", task.ID, err) @@ -1287,7 +1288,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag if confirmErr := outbox.ConfirmDurability(); confirmErr != nil { return fmt.Errorf("%v; confirm published task result: %w", journalErr, confirmErr) } - if manager != nil && isLinechainTask(task) { + if manager != nil && isLinechainTask(leased) { if cleanupErr := manager.Cleanup(task.ID, task.LeaseID); cleanupErr != nil { if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil { return fmt.Errorf("%v; cleanup confirmed linechain task: %v; upload stable result: %w", journalErr, cleanupErr, uploadErr) @@ -1301,7 +1302,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } return journalErr } - if manager != nil && isLinechainTask(task) { + if manager != nil && isLinechainTask(leased) { if err := manager.Cleanup(task.ID, task.LeaseID); err != nil { if uploadErr := flushTaskResultsRetain(cfg, outbox, true); uploadErr != nil { return fmt.Errorf("cleanup linechain task %s: %v; upload stable result: %w", task.ID, err, uploadErr) @@ -1316,9 +1317,14 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag return nil } -func isLinechainTask(task model.Task) bool { - const marker = "# lattice-linechain-e3-v1\n" - return strings.HasPrefix(task.Script, marker) && strings.Contains(task.Script, "-linechain-apply") && strings.Contains(task.Script, "LATTICE_LINECHAIN_TXN_DIR") +func isLinechainTask(task leasedAgentTask) bool { + if !task.DurableResult { + return false + } + if task.DurableProtocol != "" { + return task.DurableProtocol == "linechain-e3-v1" + } + return isLegacyLinechainTask(task.Task) } func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { @@ -1354,7 +1360,10 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo }, nil); err != nil { return fmt.Errorf("flush durable task result %s: %w", entry.Task.ID, err) } - if retain && isLinechainTask(entry.Task) { + // Pending entries predate typed lease metadata; retain only the exact + // helper namespace marker during cleanup recovery. Live classification is + // always driven by durable_protocol on the leased response. + if retain && isLegacyLinechainTask(entry.Task) { continue } if err := outbox.Remove(entry); err != nil { @@ -1364,6 +1373,10 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo return nil } +func isLegacyLinechainTask(task model.Task) bool { + return strings.HasPrefix(task.Script, "# lattice-linechain-e3-v1\n") +} + func taskResultOutboxDir(cfg agentConfig) (string, error) { base := strings.TrimSpace(cfg.TaskOutboxDir) if base == "" { diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index 1f800fc..a019bef 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -32,6 +32,7 @@ type Document struct { Version int `json:"version"` Operation string `json:"operation"` ConfigDir string `json:"config_dir"` + FragmentBasename string `json:"fragment_basename,omitempty"` FragmentPath string `json:"fragment_path"` SidecarPath string `json:"sidecar_path"` PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` @@ -245,6 +246,22 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := dec.Decode(&d); err != nil { return fmt.Errorf("decode linechain document: %w", err) } + // Artifact locations are agent-owned. The server may provide only the + // deterministic basename; full paths are derived from the locally resolved + // sing-box layout and cannot redirect writes. + if d.FragmentBasename != "" { + if filepath.Base(d.FragmentBasename) != d.FragmentBasename || strings.Contains(d.FragmentBasename, "..") || !strings.HasPrefix(d.FragmentBasename, "lattice-linechain-") || filepath.Ext(d.FragmentBasename) != ".json" { + return fmt.Errorf("fragment_basename is invalid") + } + if d.FragmentPath != "" && filepath.Clean(d.FragmentPath) != filepath.Join(m.configDir, d.FragmentBasename) { + return fmt.Errorf("fragment_path is not agent-owned") + } + d.FragmentPath = filepath.Join(m.configDir, d.FragmentBasename) + if d.SidecarPath != "" && filepath.Clean(d.SidecarPath) != filepath.Clean(m.sidecarPath) { + return fmt.Errorf("sidecar_path is not agent-owned") + } + d.SidecarPath = m.sidecarPath + } if err := m.validateDocument(d); err != nil { return err } @@ -614,7 +631,7 @@ func validateParents(path string) error { if err != nil { return err } - if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + if (!info.IsDir() && info.Mode()&os.ModeSymlink == 0) || (dir == filepath.Dir(path) && info.Mode()&os.ModeSymlink != 0) { return fmt.Errorf("artifact parent must be a real directory") } if dir == filepath.Dir(path) && !ownedPath(info) { From f5912e9ef701928aca5fa6aeaa87f33fc8d6001f Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:30:17 -0700 Subject: [PATCH 07/43] Fail closed on unknown durable task protocols Require explicit linechain-e3-v1 metadata for durable E3 execution and remove script-marker classification. Constraint: pending outbox records do not yet persist protocol metadata Rejected: Parse script comments | spoofable and conflates generic delivery Confidence: medium Scope-risk: moderate Tested: gofmt; targeted go test (legacy cleanup regression remains) Not-tested: full contract matrix and real-binary E2E --- cmd/lattice-agent/main.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 13b9ce9..f7a7f92 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1225,6 +1225,9 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag debugf(cfg, "tasks fetched: count=%d", len(tasks)) for _, leased := range tasks { task := leased.Task + if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v1" { + return fmt.Errorf("unsupported durable protocol %q", leased.DurableProtocol) + } if !leased.DurableResult { debugf(cfg, "task start without durable-result protocol: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec) result := runner.Run(task) @@ -1318,13 +1321,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } func isLinechainTask(task leasedAgentTask) bool { - if !task.DurableResult { - return false - } - if task.DurableProtocol != "" { - return task.DurableProtocol == "linechain-e3-v1" - } - return isLegacyLinechainTask(task.Task) + return task.DurableResult && task.DurableProtocol == "linechain-e3-v1" } func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { @@ -1363,7 +1360,7 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo // Pending entries predate typed lease metadata; retain only the exact // helper namespace marker during cleanup recovery. Live classification is // always driven by durable_protocol on the leased response. - if retain && isLegacyLinechainTask(entry.Task) { + if retain && false { continue } if err := outbox.Remove(entry); err != nil { @@ -1373,10 +1370,6 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo return nil } -func isLegacyLinechainTask(task model.Task) bool { - return strings.HasPrefix(task.Script, "# lattice-linechain-e3-v1\n") -} - func taskResultOutboxDir(cfg agentConfig) (string, error) { base := strings.TrimSpace(cfg.TaskOutboxDir) if base == "" { From c9c7ac22864e0773b377f02111f09a4d442895aa Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:30:52 -0700 Subject: [PATCH 08/43] Persist durable protocol with task lease journals Add an explicit protocol field and protocol-aware Begin seam so recovery can classify pending deliveries without parsing scripts. Constraint: existing Begin callers and persisted entries must remain readable Rejected: infer protocol from task script | spoofable Confidence: medium Scope-risk: narrow Tested: go test ./internal/taskoutbox Not-tested: agent integration wiring and real E2E --- internal/taskoutbox/outbox.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index 1ba085b..81cb8d5 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -35,6 +35,7 @@ type Entry struct { Version int `json:"version"` State string `json:"state"` Task model.Task `json:"task"` + DurableProtocol string `json:"durable_protocol,omitempty"` Result *model.TaskResult `json:"result,omitempty"` ExecutionStartedAt time.Time `json:"execution_started_at"` UpdatedAt time.Time `json:"updated_at"` @@ -157,6 +158,15 @@ func (s *Store) Close() error { // must not be executed again. A true result means this call published the new // lease journal, even if a subsequent directory sync reported an error. func (s *Store) Begin(task model.Task) (committed bool, err error) { + return s.begin(task, "") +} + +// BeginWithProtocol records the leased delivery discriminator for recovery. +func (s *Store) BeginWithProtocol(task model.Task, protocol string) (bool, error) { + return s.begin(task, protocol) +} + +func (s *Store) begin(task model.Task, protocol string) (committed bool, err error) { if strings.TrimSpace(task.ID) == "" || strings.TrimSpace(task.LeaseID) == "" { return false, fmt.Errorf("task id and lease id are required for durable execution") } @@ -181,6 +191,7 @@ func (s *Store) Begin(task model.Task) (committed bool, err error) { Version: entryVersion, State: stateLeased, Task: task, + DurableProtocol: protocol, ExecutionStartedAt: now, UpdatedAt: now, }) From c03139e719c529ac777877a907358fb6a7c7ace9 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:32:06 -0700 Subject: [PATCH 09/43] Block result uploads while linechain recovery is uncertain Recovery failure now gates readiness and polling without flushing pending results, preventing arbitrary durable entries from crossing an unresolved linechain authority boundary. Constraint: recovery must precede every observable delivery Rejected: flush retained results during blocked recovery | journal/outbox identity may be unresolved Confidence: high Scope-risk: moderate Tested: focused linechain tests (remaining cleanup retention assertion fails) Not-tested: full matrix and real E2E --- cmd/lattice-agent/main.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index f7a7f92..596e598 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -385,9 +385,6 @@ func main() { break } else { log.Printf("linechain recovery blocked readiness: %v", err) - if uploadErr := flushTaskResultsRetain(cfg, taskResults, true); uploadErr != nil { - log.Printf("linechain terminal result upload error: %v", uploadErr) - } time.Sleep(cfg.Interval) } } @@ -450,9 +447,6 @@ func main() { for { if err := requireLinechainRecovered(context.Background(), linechainManager, taskResults, cfg.NodeID); err != nil { log.Printf("linechain recovery blocked cycle: %v", err) - if uploadErr := flushTaskResultsRetain(cfg, taskResults, true); uploadErr != nil { - log.Printf("linechain terminal result upload error: %v", uploadErr) - } <-ticker.C continue } @@ -1194,7 +1188,6 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } if manager != nil { if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil { - _ = flushTaskResultsRetain(cfg, outbox, true) return err } } From 2f6c9629e848409f329f83b18f10b8ba7a886bcb Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:32:40 -0700 Subject: [PATCH 10/43] Carry durable protocol into lease journals Use the protocol-aware outbox seam when available while preserving compatibility with existing outbox implementations. Constraint: mocks and older stores expose only Begin(task) Rejected: change interface abruptly | breaks generic and NetGuard implementations Confidence: medium Scope-risk: narrow Tested: focused linechain test (cleanup retention remains red) Not-tested: full matrix and real E2E --- cmd/lattice-agent/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 596e598..001aeb0 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1233,7 +1233,15 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } continue } - committed, err := outbox.Begin(task) + var committed bool + var err error + if typed, ok := outbox.(interface { + BeginWithProtocol(model.Task, string) (bool, error) + }); ok { + committed, err = typed.BeginWithProtocol(task, leased.DurableProtocol) + } else { + committed, err = outbox.Begin(task) + } if err != nil { journalErr := fmt.Errorf("journal task lease %s before execution: %w", task.ID, err) if committed { From 34c6a1c762b22cf7c906cb2096d16096f060ce6b Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:33:52 -0700 Subject: [PATCH 11/43] Validate durable delivery protocol pairs before execution Treat protocol and durability as one leased-task contract, admit generic, NetGuard, and E3 pairs only, and bind protocol to duplicate journal identity. E3 tasks are rejected before journaling when linechain is unavailable. Constraint: protocol metadata is server-authored and security-sensitive Rejected: accept durable=true with missing protocol | ambiguous execution authority Confidence: high Scope-risk: moderate Tested: internal/taskoutbox passes; existing agent tests require fixture updates for explicit protocol pairs Not-tested: full contract matrix and real E2E --- cmd/lattice-agent/main.go | 23 +++++++++++++++++++++-- internal/taskoutbox/outbox.go | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 001aeb0..131584f 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1218,7 +1218,10 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag debugf(cfg, "tasks fetched: count=%d", len(tasks)) for _, leased := range tasks { task := leased.Task - if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v1" { + if err := validateDurablePair(leased, manager, cfg.LinechainReady); err != nil { + return err + } + if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v1" && leased.DurableProtocol != "netguard-v1" { return fmt.Errorf("unsupported durable protocol %q", leased.DurableProtocol) } if !leased.DurableResult { @@ -1321,6 +1324,22 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag return nil } +func validateDurablePair(task leasedAgentTask, manager *linechain.Manager, ready bool) error { + if !task.DurableResult { + if task.DurableProtocol != "" { + return fmt.Errorf("non-durable task has protocol %q", task.DurableProtocol) + } + return nil + } + if task.DurableProtocol != "linechain-e3-v1" && task.DurableProtocol != "netguard-v1" { + return fmt.Errorf("durable task has unsupported protocol %q", task.DurableProtocol) + } + if task.DurableProtocol == "linechain-e3-v1" && (manager == nil || !ready) { + return fmt.Errorf("linechain task received while durable linechain is unavailable") + } + return nil +} + func isLinechainTask(task leasedAgentTask) bool { return task.DurableResult && task.DurableProtocol == "linechain-e3-v1" } @@ -1361,7 +1380,7 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo // Pending entries predate typed lease metadata; retain only the exact // helper namespace marker during cleanup recovery. Live classification is // always driven by durable_protocol on the leased response. - if retain && false { + if retain && entry.DurableProtocol == "linechain-e3-v1" { continue } if err := outbox.Remove(entry); err != nil { diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index 81cb8d5..e74c932 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -177,7 +177,7 @@ func (s *Store) begin(task model.Task, protocol string) (committed bool, err err key := entryKey(task.ID, task.LeaseID) for _, entry := range entries { if entry.Task.ID == task.ID { - if entry.key == key && reflect.DeepEqual(entry.Task, task) { + if entry.key == key && reflect.DeepEqual(entry.Task, task) && entry.DurableProtocol == protocol { return false, nil } return false, fmt.Errorf("task %s was redelivered with a different lease or content", task.ID) From eddb76a4647bcb32d4aa1f285bfce887d2ef89a1 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:51:18 -0700 Subject: [PATCH 12/43] Require deterministic basename for linechain document v2 Accept the v2 document envelope while preserving v1 journal compatibility, and require the agent-owned fragment basename before local path derivation. Constraint: existing journals and tests remain version 1 Rejected: trust server absolute paths in v2 | host path redirection Confidence: medium Scope-risk: narrow Tested: go test ./internal/linechain Not-tested: semantic sidecar merge and real E2E --- internal/linechain/manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index a019bef..6720965 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -318,9 +318,12 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string } func (m *Manager) validateDocument(d Document) error { - if d.Version != journalVersion { + if d.Version != journalVersion && d.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", d.Version) } + if d.Version == 2 && d.FragmentBasename == "" { + return fmt.Errorf("v2 fragment_basename is required") + } switch d.Operation { case "create": if d.PreviousFragmentSHA256 != "" || d.PreviousSidecarSHA256 != "" || d.Fragment == nil || d.Sidecar == nil { From b04f66db149cbe5e383f46b4586747e44be60ac2 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:59:07 -0700 Subject: [PATCH 13/43] Reject server paths in v2 linechain documents Perform raw decode-time rejection of legacy absolute path fields before deriving agent-owned artifact locations. Constraint: v1 documents and journals retain compatibility Rejected: validate after normalization | derived local paths obscure hostile input Confidence: high Scope-risk: narrow Tested: go test ./internal/linechain Not-tested: semantic sidecar overlay and real E2E --- internal/linechain/manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index 6720965..06bb1f4 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -246,6 +246,9 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := dec.Decode(&d); err != nil { return fmt.Errorf("decode linechain document: %w", err) } + if d.Version == 2 && (d.ConfigDir != "" || d.FragmentPath != "" || d.SidecarPath != "") { + return fmt.Errorf("v2 documents must not contain server-supplied paths") + } // Artifact locations are agent-owned. The server may provide only the // deterministic basename; full paths are derived from the locally resolved // sing-box layout and cannot redirect writes. From 95269d7a72936fda54d9d4f3317c99f65a82f882 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:59:38 -0700 Subject: [PATCH 14/43] Constrain linechain artifact names to deterministic digests Require the v2 fragment basename to match the exact lowercase hexadecimal namespace contract. Constraint: server must bind deterministic artifact identity Rejected: broad prefix and extension checks | permits ambiguous names Confidence: high Scope-risk: narrow Tested: go test ./internal/linechain Not-tested: sidecar overlay and real E2E --- internal/linechain/manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index 06bb1f4..7df2a2c 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -26,6 +27,8 @@ const ( maxDiagnostic = 4096 ) +var linechainBasenameRE = regexp.MustCompile(`^lattice-linechain-[0-9a-f]{20}\.json$`) + // Document is the bounded server-rendered input consumed by -linechain-apply. // Desired content is never copied into the journal; only its digest is stored. type Document struct { @@ -253,7 +256,7 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string // deterministic basename; full paths are derived from the locally resolved // sing-box layout and cannot redirect writes. if d.FragmentBasename != "" { - if filepath.Base(d.FragmentBasename) != d.FragmentBasename || strings.Contains(d.FragmentBasename, "..") || !strings.HasPrefix(d.FragmentBasename, "lattice-linechain-") || filepath.Ext(d.FragmentBasename) != ".json" { + if filepath.Base(d.FragmentBasename) != d.FragmentBasename || !linechainBasenameRE.MatchString(d.FragmentBasename) { return fmt.Errorf("fragment_basename is invalid") } if d.FragmentPath != "" && filepath.Clean(d.FragmentPath) != filepath.Join(m.configDir, d.FragmentBasename) { From 3779d348c75d15565fbf6fa041300e4636ae0bf9 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:52:34 -0700 Subject: [PATCH 15/43] Prove line-chain routing against the real runtime Replace the file-write smoke test with a loopback-only sing-box 1.13.x harness that observes the B-to-A hop, drives basename-only v2 Manager documents through create/resync/remove, validates runtime discovery, and exercises process-group crash recovery while the supervisor remains alive. Keep the binary external and mandatory in the script lane. Constraint: E2E owns only the agent test and invocation script; Manager, main loop, and outbox integration remain worker-2 owned Constraint: The official sing-box binary is a local test tool, never a Go or repository dependency Rejected: Stub restart/check commands | they cannot prove that B loads the published fragment Rejected: Starting B pre-chained | it cannot prove the apply operation changed live routing Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep the v2 fixture basename-only and preserve real process restart plus observer assertions Tested: go test ./cmd/lattice-agent -run '^TestLinechainE2E(Apply|Restart|Active)Helper$' -count=1 Not-tested: Full real-binary lane pending integration with worker-2 final v2 Manager head --- cmd/lattice-agent/linechain_e2e_test.go | 546 ++++++++++++++++++++++-- scripts/test-linechain-e2e.sh | 15 +- 2 files changed, 517 insertions(+), 44 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index 9329b54..a973139 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -3,7 +3,9 @@ package main import ( "bytes" "context" + "crypto/sha256" "encoding/binary" + "encoding/hex" "encoding/json" "fmt" "io" @@ -11,61 +13,358 @@ import ( "os" "os/exec" "path/filepath" + "strconv" + "strings" + "syscall" "testing" "time" "github.com/LatticeNet/lattice-node-agent/internal/linechain" + "github.com/LatticeNet/lattice-node-agent/internal/singboxdiscover" + "github.com/LatticeNet/lattice-sdk/model" ) -// TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. -// Normal unit runs leave the real-binary lane to that mandatory harness. +const ( + linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN" + linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT" + linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR" + linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR" + linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT" + linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" + linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" +) + +// TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. The +// script makes the real 1.13.x binary mandatory, so this test never skips. func TestLinechainRealSingBoxE2E(t *testing.T) { - bin := os.Getenv("LATTICE_SINGBOX_E2E_BIN") - if bin == "" { - return - } + bin := requireSingBoxE2EBinary(t) root := t.TempDir() origin := startEchoOrigin(t) + observer := startTCPObserver(t, origin.String()) aPort, bPort, clientPort := freePort(t), freePort(t), freePort(t) - const uuidA = "11111111-1111-4111-8111-111111111111" - const uuidB = "22222222-2222-4222-8222-222222222222" - aConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":"%s"}]}],"outbounds":[{"type":"direct","tag":"direct"}],"route":{"final":"direct"}}`, aPort, uuidA) - bConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"vless","tag":"source-b","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":"%s"}]}],"outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":"%s"}],"route":{"final":"chain-to-a"}}`, bPort, uuidB, aPort, uuidA) - clientConfig := fmt.Sprintf(`{"log":{"level":"error"},"inbounds":[{"type":"socks","tag":"client","listen":"127.0.0.1","listen_port":%d}],"outbounds":[{"type":"vless","tag":"to-b","server":"127.0.0.1","server_port":%d,"uuid":"%s"}],"route":{"final":"to-b"}}`, clientPort, bPort, uuidB) - startSingBox(t, bin, root, "a", aConfig, aPort) - startSingBox(t, bin, root, "b", bConfig, bPort) - startSingBox(t, bin, root, "client", clientConfig, clientPort) + const ( + uuidA = "11111111-1111-4111-8111-111111111111" + uuidB = "22222222-2222-4222-8222-222222222222" + lineUUIDA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + lineUUIDB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + basename = "lattice-linechain-0123456789abcdef0123.json" + ) + + aDir := filepath.Join(root, "a") + bDir := filepath.Join(root, "b") + clientDir := filepath.Join(root, "client") + mustMkdir(t, aDir) + mustMkdir(t, bDir) + mustMkdir(t, clientDir) + writeFile(t, filepath.Join(aDir, "config.json"), fmt.Sprintf(`{ + "log":{"level":"error"}, + "inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q}]}], + "outbounds":[{"type":"direct","tag":"direct"}], + "route":{"rules":[{"inbound":["target-a"],"outbound":"direct"}]} +} +`, aPort, uuidA)) + writeFile(t, filepath.Join(bDir, "config.json"), fmt.Sprintf(`{ + "log":{"level":"error"}, + "inbounds":[{"type":"vless","tag":"source-b","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q}]}], + "outbounds":[{"type":"direct","tag":"direct"}], + "route":{"final":"direct"} +} +`, bPort, uuidB)) + writeFile(t, filepath.Join(clientDir, "config.json"), fmt.Sprintf(`{ + "log":{"level":"error"}, + "inbounds":[{"type":"socks","tag":"client","listen":"127.0.0.1","listen_port":%d}], + "outbounds":[{"type":"vless","tag":"to-b","server":"127.0.0.1","server_port":%d,"uuid":%q}], + "route":{"rules":[{"inbound":["client"],"outbound":"to-b"}]} +} +`, clientPort, bPort, uuidB)) + + startManagedSingBox(t, bin, root, "a", aDir, aPort) + startManagedSingBox(t, bin, root, "b", bDir, bPort) + startManagedSingBox(t, bin, root, "client", clientDir, clientPort) + assertSOCKSEcho(t, clientPort, origin) + if got := observer.Count(); got != 0 { + t.Fatalf("B began chained: observer accepted %d connections before apply", got) + } - conf := filepath.Join(root, "conf") - if err := os.Mkdir(conf, 0o700); err != nil { + sidecarPath := filepath.Join(root, "lattice-metadata.json") + fragmentPath := filepath.Join(bDir, basename) + txnDir := filepath.Join(root, "txn") + fragment := fmt.Sprintf(`{ + "outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":%q}], + "route":{"rules":[{"inbound":["source-b"],"outbound":"chain-to-a"}]} +} +`, observer.Port(), uuidA) + sidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} +`, lineUUIDB, lineUUIDA) + + m := openE2EManager(t, bin, root, bDir, sidecarPath, txnDir, bPort) + defer m.Close() + + // Crash a real apply helper process group after it publishes both artifacts, + // while this test (the agent/supervisor analogue) remains alive. Recovery + // must restore and restart B before any inventory, traffic, or result callback. + crashDoc := bindE2EDocument("create", basename, "", "", &fragment, &sidecar) + crashBytes := marshalE2EDocument(t, crashDoc) + apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--") + apply.Env = append(os.Environ(), + linechainE2EBinEnv+"="+bin, + linechainE2ERootEnv+"="+root, + linechainE2EConfigEnv+"="+bDir, + linechainE2ESidecarEnv+"="+sidecarPath, + linechainE2EBPortEnv+"="+strconv.Itoa(bPort), + linechainE2ETaskEnv+"=crash-task", + linechainE2ELeaseEnv+"=crash-lease", + ) + apply.Stdin = bytes.NewReader(crashBytes) + apply.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + var applyLog bytes.Buffer + apply.Stdout, apply.Stderr = &applyLog, &applyLog + if err := apply.Start(); err != nil { t.Fatal(err) } - fragmentPath := filepath.Join(conf, "lattice-linechain-e2e.json") - sidecarPath := filepath.Join(root, "lattice-metadata.json") - fragment := "{\n \"outbounds\": [{\"type\": \"direct\", \"tag\": \"linechain-direct\"}]\n}\n" - sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}\n" - doc := linechain.BindDocument(linechain.Document{Version: 1, Operation: "create", ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) - b, err := json.Marshal(doc) + waitForExactPair(t, fragmentPath, fragment, sidecarPath, sidecar) + if err := syscall.Kill(-apply.Process.Pid, syscall.SIGKILL); err != nil { + t.Fatalf("kill apply helper process group: %v", err) + } + _ = apply.Wait() + + order := []string{} + if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { + order = append(order, "result") + if result.ExitCode == 0 || !strings.Contains(result.Error, "interrupted") { + return fmt.Errorf("unexpected recovered result: %+v", result) + } + return nil + }, "node-b"); err != nil { + t.Fatalf("recover killed helper: %v (helper output %s)", err, applyLog.String()) + } + order = append(order, "inventory") + assertUnchainedInventory(t, bDir, sidecarPath) + order = append(order, "traffic") + assertSOCKSEcho(t, clientPort, origin) + if strings.Join(order, ",") != "result,inventory,traffic" { + t.Fatalf("recovery ordering = %v", order) + } + if got := observer.Count(); got != 0 { + t.Fatalf("recovery exposed chained traffic before successful apply: %d", got) + } + + applyAndResolve(t, m, marshalE2EDocument(t, crashDoc), "create-task", "create-lease") + assertSOCKSEcho(t, clientPort, origin) + if got := observer.Count(); got == 0 { + t.Fatal("traffic did not traverse the B-to-A observer after apply") + } + assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) + + // An ordinary metadata rewrite is a replace of the same fragment plus an + // augmented sidecar. The declared edge must survive the resync exactly. + resyncedSidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","generated_by":"ordinary-resync","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} +`, lineUUIDB, lineUUIDA) + replaceDoc := bindE2EDocument("replace", basename, digestText(fragment), digestText(sidecar), &fragment, &resyncedSidecar) + applyAndResolve(t, m, marshalE2EDocument(t, replaceDoc), "resync-task", "resync-lease") + assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) + + removedSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]} +` + removeDoc := bindE2EDocument("remove", basename, digestText(fragment), digestText(resyncedSidecar), nil, &removedSidecar) + applyAndResolve(t, m, marshalE2EDocument(t, removeDoc), "remove-task", "remove-lease") + if _, err := os.Stat(fragmentPath); !os.IsNotExist(err) { + t.Fatalf("removed fragment still exists: %v", err) + } + assertUnchainedInventory(t, bDir, sidecarPath) + before := observer.Count() + assertSOCKSEcho(t, clientPort, origin) + if got := observer.Count(); got != before { + t.Fatalf("traffic still used A after remove: observer %d -> %d", before, got) + } +} + +// TestLinechainE2EApplyHelper is a child-only entry point used to create a real +// crash boundary. The parent test kills this process group, not the supervisor. +func TestLinechainE2EApplyHelper(t *testing.T) { + if os.Getenv(linechainE2ETaskEnv) == "" { + return + } + m, err := linechain.OpenHelper(filepath.Join(os.Getenv(linechainE2ERootEnv), "txn")) + if err != nil { + t.Fatal(err) + } + defer m.Close() + configureE2EManager(t, m) + if err := m.Apply(context.Background(), os.Stdin, os.Getenv(linechainE2ETaskEnv), os.Getenv(linechainE2ELeaseEnv)); err != nil { + t.Fatal(err) + } +} + +// TestLinechainE2ERestartHelper is the fixed restart/active command used by the +// Manager. It stops the prior B instance and starts a checked replacement. +func TestLinechainE2ERestartHelper(t *testing.T) { + root := os.Getenv(linechainE2ERootEnv) + if root == "" { + return + } + if err := restartManagedSingBox(os.Getenv(linechainE2EBinEnv), root, "b", os.Getenv(linechainE2EConfigEnv), mustEnvPort(linechainE2EBPortEnv)); err != nil { + t.Fatal(err) + } +} + +func TestLinechainE2EActiveHelper(t *testing.T) { + root := os.Getenv(linechainE2ERootEnv) + if root == "" { + return + } + if err := verifyManagedSingBox(root, "b", mustEnvPort(linechainE2EBPortEnv)); err != nil { + t.Fatal(err) + } +} + +type e2eDocument map[string]any + +func bindE2EDocument(operation, basename, previousFragment, previousSidecar string, fragment, sidecar *string) e2eDocument { + d := e2eDocument{"version": 2, "operation": operation, "fragment_basename": basename} + if previousFragment != "" { + d["previous_fragment_sha256"] = previousFragment + } + if previousSidecar != "" { + d["previous_sidecar_sha256"] = previousSidecar + } + if fragment != nil { + d["fragment"] = *fragment + } + if sidecar != nil { + d["sidecar"] = *sidecar + } + fragmentDigest, sidecarDigest := digestPointer(fragment), digestPointer(sidecar) + d["fragment_sha256"], d["sidecar_sha256"] = fragmentDigest, sidecarDigest + pair := []byte{} + if fragment != nil { + pair = append(pair, *fragment...) + } + pair = append(pair, 0) + if sidecar != nil { + pair = append(pair, *sidecar...) + } + d["combined_sha256"] = digestBytes(pair) + return d +} + +func marshalE2EDocument(t *testing.T, d e2eDocument) []byte { + t.Helper() + b, err := json.Marshal(d) if err != nil { t.Fatal(err) } - m, err := linechain.Open(filepath.Join(root, "txn")) + return b +} + +func openE2EManager(t *testing.T, bin, root, configDir, sidecarPath, txnDir string, bPort int) *linechain.Manager { + t.Helper() + for key, value := range map[string]string{ + linechainE2EBinEnv: bin, linechainE2ERootEnv: root, linechainE2EConfigEnv: configDir, + linechainE2ESidecarEnv: sidecarPath, linechainE2EBPortEnv: strconv.Itoa(bPort), + } { + t.Setenv(key, value) + } + m, err := linechain.Open(txnDir) if err != nil { t.Fatal(err) } - defer m.Close() - if err := m.ConfigureLayout(conf, sidecarPath); err != nil { + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + configureE2EManager(t, m) + return m +} + +func configureE2EManager(t *testing.T, m *linechain.Manager) { + t.Helper() + if err := m.ConfigureLayout(os.Getenv(linechainE2EConfigEnv), os.Getenv(linechainE2ESidecarEnv)); err != nil { + t.Fatal(err) + } + restart := []string{os.Args[0], "-test.run=^TestLinechainE2ERestartHelper$", "--"} + verify := []string{os.Args[0], "-test.run=^TestLinechainE2EActiveHelper$", "--"} + if err := m.ConfigureCommands(os.Getenv(linechainE2EBinEnv), restart, verify); err != nil { + t.Fatal(err) + } +} + +func applyAndResolve(t *testing.T, m *linechain.Manager, document []byte, taskID, leaseID string) { + t.Helper() + if err := m.Apply(context.Background(), bytes.NewReader(document), taskID, leaseID); err != nil { + t.Fatal(err) + } + result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0}) + if err != nil || result.ExitCode != 0 || result.Error != "" { + t.Fatalf("resolve %s: result=%+v err=%v", taskID, result, err) + } + if err := m.Cleanup(taskID, leaseID); err != nil { + t.Fatalf("cleanup %s: %v", taskID, err) + } +} + +func assertChainedInventory(t *testing.T, configDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA string, observerPort int) { + t.Helper() + inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", []string{filepath.Join(configDir, "config.json"), fragmentPath}, sidecarPath) + if err != nil { t.Fatal(err) } - if err := m.ConfigureCommands(bin, []string{"true"}, []string{"true"}); err != nil { + n := findInventoryNode(t, inv, "source-b") + if n.OutboundRef != "chain-to-a" || n.OutboundServer != "127.0.0.1" || n.OutboundPort != strconv.Itoa(observerPort) || n.OutboundType != "vless" { + t.Fatalf("discovered outbound identity mismatch: %+v", n) + } + if n.LineUUID != lineUUIDB || n.DownstreamLineUUID != lineUUIDA { + t.Fatalf("discovered sidecar identity mismatch: %+v", n) + } +} + +func assertUnchainedInventory(t *testing.T, configDir, sidecarPath string) { + t.Helper() + inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", []string{filepath.Join(configDir, "config.json")}, sidecarPath) + if err != nil { t.Fatal(err) } - if err := m.Apply(context.Background(), bytes.NewReader(b), "e2e-task", "e2e-lease"); err != nil { + n := findInventoryNode(t, inv, "source-b") + if n.OutboundRef != "" || n.OutboundServer != "" || n.OutboundPort != "" || n.DownstreamLineUUID != "" { + t.Fatalf("expected unchained inventory, got %+v", n) + } +} + +func findInventoryNode(t *testing.T, inv model.SingBoxInventory, name string) model.SingBoxNode { + t.Helper() + for _, n := range inv.Nodes { + if n.Name == name { + return n + } + } + t.Fatalf("inventory lacks %q: %+v", name, inv.Nodes) + return model.SingBoxNode{} +} + +func requireSingBoxE2EBinary(t *testing.T) string { + t.Helper() + bin := os.Getenv(linechainE2EBinEnv) + if bin == "" || !filepath.IsAbs(bin) { + t.Fatalf("%s must name an absolute official sing-box 1.13.x binary", linechainE2EBinEnv) + } + out, err := exec.Command(bin, "version").CombinedOutput() + if err != nil || !strings.Contains(string(out), "sing-box version 1.13.") { + t.Fatalf("%s is not sing-box 1.13.x: %v\n%s", bin, err, out) + } + return bin +} + +func mustMkdir(t *testing.T, path string) { + t.Helper() + if err := os.Mkdir(path, 0o700); err != nil { t.Fatal(err) } - if got, err := os.ReadFile(fragmentPath); err != nil || string(got) != fragment { - t.Fatalf("fragment mismatch: %q %v", got, err) +} + +func writeFile(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) } } @@ -98,29 +397,181 @@ func startEchoOrigin(t *testing.T) *net.TCPAddr { return l.Addr().(*net.TCPAddr) } -func startSingBox(t *testing.T, bin, root, name, config string, port int) { +type tcpObserver struct { + listener net.Listener + count chan struct{} +} + +func startTCPObserver(t *testing.T, target string) *tcpObserver { t.Helper() - path := filepath.Join(root, name+".json") - if err := os.WriteFile(path, []byte(config), 0o600); err != nil { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { t.Fatal(err) } - cmd := exec.Command(bin, "run", "-c", path) - var stderr bytes.Buffer - cmd.Stderr = &stderr - if err := cmd.Start(); err != nil { + o := &tcpObserver{listener: l, count: make(chan struct{}, 128)} + t.Cleanup(func() { _ = l.Close() }) + go func() { + for { + incoming, err := l.Accept() + if err != nil { + return + } + o.count <- struct{}{} + go func() { + defer incoming.Close() + upstream, err := net.Dial("tcp", target) + if err != nil { + return + } + defer upstream.Close() + go func() { _, _ = io.Copy(upstream, incoming); _ = upstream.(*net.TCPConn).CloseWrite() }() + _, _ = io.Copy(incoming, upstream) + }() + } + }() + return o +} + +func (o *tcpObserver) Port() int { return o.listener.Addr().(*net.TCPAddr).Port } +func (o *tcpObserver) Count() int { + n := 0 + for { + select { + case <-o.count: + n++ + default: + return n + } + } +} + +func startManagedSingBox(t *testing.T, bin, root, name, configDir string, port int) { + t.Helper() + if err := restartManagedSingBox(bin, root, name, configDir, port); err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) - deadline := time.Now().Add(5 * time.Second) + t.Cleanup(func() { _ = killManagedSingBox(root, name) }) +} + +func restartManagedSingBox(bin, root, name, configDir string, port int) error { + if err := killManagedSingBox(root, name); err != nil { + return err + } + logPath := filepath.Join(root, name+".log") + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + cmd := exec.Command(bin, "run", "-C", configDir) + cmd.Stdout, cmd.Stderr = logFile, logFile + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + _ = logFile.Close() + return err + } + _ = logFile.Close() + if err := os.WriteFile(filepath.Join(root, name+".pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + return err + } + if err := waitForPort(port, 8*time.Second); err != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + logBytes, _ := os.ReadFile(logPath) + return fmt.Errorf("start %s: %w: %s", name, err, logBytes) + } + return nil +} + +func killManagedSingBox(root, name string) error { + pidPath := filepath.Join(root, name+".pid") + raw, err := os.ReadFile(pidPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || pid <= 1 { + return fmt.Errorf("invalid %s pid %q", name, raw) + } + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH { + return err + } + deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { - c, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 50*time.Millisecond) + if err := syscall.Kill(pid, 0); err == syscall.ESRCH { + return nil + } + time.Sleep(20 * time.Millisecond) + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { + return err + } + return nil +} + +func verifyManagedSingBox(root, name string, port int) error { + raw, err := os.ReadFile(filepath.Join(root, name+".pid")) + if err != nil { + return err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || pid <= 1 { + return fmt.Errorf("invalid pid %q", raw) + } + if err := syscall.Kill(pid, 0); err != nil { + return fmt.Errorf("sing-box process inactive: %w", err) + } + return waitForPort(port, time.Second) +} + +func waitForPort(port int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + address := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) + for time.Now().Before(deadline) { + c, err := net.DialTimeout("tcp", address, 50*time.Millisecond) if err == nil { _ = c.Close() - return + return nil } time.Sleep(25 * time.Millisecond) } - t.Fatalf("sing-box %s did not listen: %s", name, stderr.String()) + return fmt.Errorf("port %d did not become ready", port) +} + +func waitForExactPair(t *testing.T, fragmentPath, fragment, sidecarPath, sidecar string) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + f, ferr := os.ReadFile(fragmentPath) + s, serr := os.ReadFile(sidecarPath) + if ferr == nil && serr == nil && string(f) == fragment && string(s) == sidecar { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("apply helper did not publish the exact pair before crash boundary") +} + +func mustEnvPort(key string) int { + port, err := strconv.Atoi(os.Getenv(key)) + if err != nil || port < 1 || port > 65535 { + panic("invalid " + key) + } + return port +} + +func digestText(s string) string { return digestBytes([]byte(s)) } +func digestPointer(s *string) string { + if s == nil { + return "" + } + return digestText(*s) +} +func digestBytes(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) } func assertSOCKSEcho(t *testing.T, socksPort int, target *net.TCPAddr) { @@ -164,8 +615,17 @@ func assertSOCKSEcho(t *testing.T, socksPort int, target *net.TCPAddr) { if _, err = c.Write(payload); err != nil { t.Fatal(err) } - got := make([]byte, len(payload)) - if _, err = io.ReadFull(c, got); err != nil || !bytes.Equal(got, payload) { + got := make([]byte, 0, 128) + buf := make([]byte, 32) + for len(got) < cap(got) && !bytes.Contains(got, payload) { + n, readErr := c.Read(buf) + got = append(got, buf[:n]...) + if readErr != nil { + err = readErr + break + } + } + if !bytes.Contains(got, payload) { t.Fatalf("chain echo = %q err=%v", got, err) } } diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 9781ba7..b1e7737 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -5,6 +5,19 @@ bin="${LATTICE_SINGBOX_E2E_BIN:-}" [ -n "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN is required" >&2; exit 1; } case "$bin" in /*) ;; *) echo "LATTICE_SINGBOX_E2E_BIN must be absolute" >&2; exit 1;; esac [ -x "$bin" ] || { echo "LATTICE_SINGBOX_E2E_BIN must be executable" >&2; exit 1; } -"$bin" version >/dev/null 2>&1 || { echo "sing-box version failed" >&2; exit 1; } +version="$($bin version 2>&1)" || { echo "sing-box version failed" >&2; exit 1; } +printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || { + echo "LATTICE_SINGBOX_E2E_BIN must be official sing-box 1.13.x" >&2 + printf '%s\n' "$version" >&2 + exit 1 +} + +probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")" +trap 'find "$probe" -type f -delete 2>/dev/null || true; rmdir "$probe" 2>/dev/null || true' EXIT HUP INT TERM +cat >"$probe/config.json" <<'JSON' +{"inbounds":[],"outbounds":[{"type":"direct","tag":"direct"}],"route":{"final":"direct"}} +JSON +"$bin" check -C "$probe" >/dev/null 2>&1 || { echo "sing-box 1.13.x config check failed" >&2; exit 1; } +printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')" LATTICE_SINGBOX_E2E_BIN="$bin" go test ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From 6ba982ecc22b8697e28b64228825474137efacea Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:04:19 -0700 Subject: [PATCH 16/43] Pin E3 protocol in handoff regression fixture Update the cleanup-failure regression to exercise the explicit leased E3 protocol contract rather than obsolete durable-result-only payloads. Constraint: generic and NetGuard deliveries must remain distinct Rejected: infer E3 from script marker | spoofable Confidence: high Scope-risk: narrow Tested: focused test remains red, exposing outbox completion protocol propagation gap Not-tested: full matrix and real E2E --- cmd/lattice-agent/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index 98fc36c..1969974 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -381,7 +381,7 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { case "/api/agent/tasks": fetches++ if fetches == 1 { - b, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + b, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v1"}}) return testResponse(http.StatusOK, string(b)), nil } return testResponse(http.StatusOK, "[]"), nil From 9a09948c4ca2ca81be79693c242a8518910c37dc Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:08:44 -0700 Subject: [PATCH 17/43] Enable linechain readiness in E3 handoff regression The integration regression must model a node that resolved its local sing-box layout; otherwise the protocol guard correctly rejects the lease before exercising retention. Constraint: E3 execution is forbidden when readiness is false Rejected: weaken production guard | would permit unsafe host execution Confidence: high Scope-risk: narrow Tested: go test ./cmd/lattice-agent -run TestRunTasksLinechainCompletesHandoffWithoutReplay -count=1 Not-tested: full matrix and real E2E --- cmd/lattice-agent/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index 1969974..3f20092 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -399,7 +399,7 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { return testResponse(http.StatusNotFound, ""), nil } })} - cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"} + cfg := agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret", LinechainReady: true} if err := runTasks(cfg, runner, outbox, manager); err == nil { t.Fatal("cleanup failure should remain readiness-visible") } From 5fdae5a2fd8232bde7f4bde984844abedfcdb59b Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 06:52:34 -0700 Subject: [PATCH 18/43] Prove line-chain routing against the real runtime Replace the file-write smoke test with a loopback-only sing-box 1.13.x harness that observes the B-to-A hop, drives basename-only v2 Manager documents through create/resync/remove, validates runtime discovery, and exercises process-group crash recovery while the supervisor remains alive. Keep the binary external and mandatory in the script lane. Constraint: E2E owns only the agent test and invocation script; Manager, main loop, and outbox integration remain worker-2 owned Constraint: The official sing-box binary is a local test tool, never a Go or repository dependency Rejected: Stub restart/check commands | they cannot prove that B loads the published fragment Rejected: Starting B pre-chained | it cannot prove the apply operation changed live routing Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep the v2 fixture basename-only and preserve real process restart plus observer assertions Tested: go test ./cmd/lattice-agent -run '^TestLinechainE2E(Apply|Restart|Active)Helper$' -count=1 Not-tested: Full real-binary lane pending integration with worker-2 final v2 Manager head --- cmd/lattice-agent/linechain_e2e_test.go | 2 ++ scripts/test-linechain-e2e.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index a973139..d9a7722 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -1,3 +1,5 @@ +//go:build linechain_e2e && (darwin || linux || freebsd || openbsd || netbsd) + package main import ( diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index b1e7737..8923ad6 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -20,4 +20,4 @@ JSON "$bin" check -C "$probe" >/dev/null 2>&1 || { echo "sing-box 1.13.x config check failed" >&2; exit 1; } printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')" -LATTICE_SINGBOX_E2E_BIN="$bin" go test ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v +LATTICE_SINGBOX_E2E_BIN="$bin" go test -tags=linechain_e2e ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From beb8c19f1df84d4a7891969a0a257708fc6e693c Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:10:53 -0700 Subject: [PATCH 19/43] Refuse generic recovery of leased E3 journals A leased linechain task must be resolved by the linechain journal manager before generic interrupted-result synthesis; generic and NetGuard entries retain legacy recovery behavior. Constraint: E3 execution outcome is owned by the linechain transaction journal Rejected: synthesize unknown result in generic outbox path | can conflict with terminal journal Confidence: high Scope-risk: moderate Tested: go test ./internal/taskoutbox Not-tested: full cross-check matrix and real E2E --- internal/taskoutbox/outbox.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index e74c932..ec84704 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -250,6 +250,9 @@ func (s *Store) RecoverInterrupted(nodeID string) error { if entry.State != stateLeased { continue } + if entry.DurableProtocol == "linechain-e3-v1" { + return fmt.Errorf("leased E3 task %s requires linechain journal recovery before generic outbox recovery", entry.Task.ID) + } now := time.Now().UTC() result := model.TaskResult{ TaskID: entry.Task.ID, From 7143bc653e2a1b32fdfbaa937d391618ec3e6a23 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:23:10 -0700 Subject: [PATCH 20/43] Make E3 artifact authority local and sidecar-tolerant Decode only the basename-bound v2 wire contract, derive all host paths from the configured runtime layout, and keep ordinary sidecar rewrites outside the fragment predecessor CAS. Normalize absent artifact digests consistently so create recovery can restore the exact old pair. Constraint: E3 v2 wire documents cannot carry host paths or previous sidecar digests Constraint: Existing v1 test fixtures are obsolete and migrate with the strict decoder Rejected: Preserve v1 wire compatibility | the approved E3 contract requires strict v2-only decode Confidence: high Scope-risk: moderate Reversibility: clean Directive: Never reintroduce server-controlled artifact paths or sidecar predecessor CAS into v2 Tested: go test ./internal/linechain ./internal/taskoutbox ./cmd/lattice-agent -count=1 Not-tested: Real sing-box lane until accepted E2E increment is integrated --- cmd/lattice-agent/main_test.go | 23 ++++++----- internal/linechain/manager.go | 63 ++++++++++++++++-------------- internal/linechain/manager_test.go | 23 ++++++----- 3 files changed, 61 insertions(+), 48 deletions(-) diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index 3f20092..d3f682e 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -264,7 +264,7 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { } fetchCalls++ if fetchCalls == 1 { - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) return testResponse(http.StatusOK, string(data)), nil } return testResponse(http.StatusOK, `[]`), nil @@ -355,12 +355,15 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - fragmentPath := filepath.Join(conf, "lattice-linechain-chain.json") - sidecarPath := filepath.Join(root, "lattice-metadata.json") + fragmentPath := filepath.Join(conf, "lattice-linechain-0123456789abcdef0123.json") fragment := "{\"outbounds\":[]}" sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}" - docValue := linechain.BindDocument(linechain.Document{Version: 1, Operation: "create", ConfigDir: conf, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) - doc, err := json.Marshal(docValue) + docValue := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, Sidecar: &sidecar}) + doc, err := json.Marshal(map[string]any{ + "version": docValue.Version, "operation": docValue.Operation, "fragment_basename": docValue.FragmentBasename, + "fragment": docValue.Fragment, "sidecar": docValue.Sidecar, "fragment_sha256": docValue.FragmentSHA256, + "sidecar_sha256": docValue.SidecarSHA256, "combined_sha256": docValue.CombinedSHA256, + }) if err != nil { t.Fatal(err) } @@ -532,7 +535,7 @@ func TestRunTasksJournalFailurePreventsExecution(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must not run"} - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) var reported model.TaskResult httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { if r.URL.Path == "/api/agent/tasks" { @@ -569,7 +572,7 @@ func TestRunTasksPublishedJournalFailureDoesNotPostConflictingDirectResult(t *te oldClient := httpClient defer func() { httpClient = oldClient }() task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must not run"} - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) postCalls := 0 httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { if r.URL.Path == "/api/agent/tasks" { @@ -596,7 +599,7 @@ func TestRunTasksExactRedeliveryDoesNotExecuteExistingJournal(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "must run once"} - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { if r.URL.Path != "/api/agent/tasks" { t.Fatalf("unexpected request for already-journaled lease: %s", r.URL.Path) @@ -688,7 +691,7 @@ func TestRunTasksConfirmsAndUploadsResultPublishedBeforeDirectorySyncFailure(t * oldClient := httpClient defer func() { httpClient = oldClient }() task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo once"} - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) posts := 0 var posted model.TaskResult outbox := &completePublishingOutbox{} @@ -730,7 +733,7 @@ func TestRunTasksDoesNotUploadUnconfirmedPublishedResult(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() task := model.Task{ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo once"} - data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true}}) + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "netguard-v1"}}) posts := 0 httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { switch r.URL.Path { diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index 7df2a2c..ec9216a 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -47,6 +47,21 @@ type Document struct { CombinedSHA256 string `json:"combined_sha256"` } +// wireDocumentV2 is the only server-controlled shape accepted by Apply. +// Host paths and the ordinary-writer sidecar predecessor are intentionally not +// representable on the wire; Apply derives paths from the local layout. +type wireDocumentV2 struct { + Version int `json:"version"` + Operation string `json:"operation"` + FragmentBasename string `json:"fragment_basename"` + PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` + Fragment *string `json:"fragment,omitempty"` + Sidecar *string `json:"sidecar,omitempty"` + FragmentSHA256 string `json:"fragment_sha256,omitempty"` + SidecarSHA256 string `json:"sidecar_sha256,omitempty"` + CombinedSHA256 string `json:"combined_sha256"` +} + // BindDocument computes the deterministic desired artifact binding. func BindDocument(d Document) Document { d.FragmentSHA256 = digestPtr(d.Fragment) @@ -245,28 +260,21 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string } dec := json.NewDecoder(io.LimitReader(r, maxDocumentSize+1)) dec.DisallowUnknownFields() - var d Document - if err := dec.Decode(&d); err != nil { + var wire wireDocumentV2 + if err := dec.Decode(&wire); err != nil { return fmt.Errorf("decode linechain document: %w", err) } - if d.Version == 2 && (d.ConfigDir != "" || d.FragmentPath != "" || d.SidecarPath != "") { - return fmt.Errorf("v2 documents must not contain server-supplied paths") + if wire.Version != 2 { + return fmt.Errorf("unsupported linechain document version %d", wire.Version) } - // Artifact locations are agent-owned. The server may provide only the - // deterministic basename; full paths are derived from the locally resolved - // sing-box layout and cannot redirect writes. - if d.FragmentBasename != "" { - if filepath.Base(d.FragmentBasename) != d.FragmentBasename || !linechainBasenameRE.MatchString(d.FragmentBasename) { - return fmt.Errorf("fragment_basename is invalid") - } - if d.FragmentPath != "" && filepath.Clean(d.FragmentPath) != filepath.Join(m.configDir, d.FragmentBasename) { - return fmt.Errorf("fragment_path is not agent-owned") - } - d.FragmentPath = filepath.Join(m.configDir, d.FragmentBasename) - if d.SidecarPath != "" && filepath.Clean(d.SidecarPath) != filepath.Clean(m.sidecarPath) { - return fmt.Errorf("sidecar_path is not agent-owned") - } - d.SidecarPath = m.sidecarPath + if filepath.Base(wire.FragmentBasename) != wire.FragmentBasename || !linechainBasenameRE.MatchString(wire.FragmentBasename) { + return fmt.Errorf("fragment_basename is invalid") + } + d := Document{ + Version: wire.Version, Operation: wire.Operation, ConfigDir: m.configDir, + FragmentBasename: wire.FragmentBasename, FragmentPath: filepath.Join(m.configDir, wire.FragmentBasename), SidecarPath: m.sidecarPath, + PreviousFragmentSHA256: wire.PreviousFragmentSHA256, Fragment: wire.Fragment, Sidecar: wire.Sidecar, + FragmentSHA256: wire.FragmentSHA256, SidecarSHA256: wire.SidecarSHA256, CombinedSHA256: wire.CombinedSHA256, } if err := m.validateDocument(d); err != nil { return err @@ -287,11 +295,8 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := requirePrevious("fragment", fragmentOld, fragmentHad, d.PreviousFragmentSHA256); err != nil { return err } - if err := requirePrevious("sidecar", sidecarOld, sidecarHad, d.PreviousSidecarSHA256); err != nil { - return err - } j := journal{Version: journalVersion, TaskID: taskID, LeaseID: leaseID, FragmentPath: d.FragmentPath, SidecarPath: d.SidecarPath, - FragmentOld: digest(fragmentOld), SidecarOld: digest(sidecarOld), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad, + FragmentOld: digestMaybe(fragmentOld, fragmentHad), SidecarOld: digestMaybe(sidecarOld, sidecarHad), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad, FragmentNew: digestPtr(d.Fragment), SidecarNew: digestPtr(d.Sidecar), Phase: "prepared"} if err := m.writeBackup(path+".fragment.old", fragmentOld, fragmentHad); err != nil { return err @@ -324,23 +329,23 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string } func (m *Manager) validateDocument(d Document) error { - if d.Version != journalVersion && d.Version != 2 { + if d.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", d.Version) } - if d.Version == 2 && d.FragmentBasename == "" { + if d.FragmentBasename == "" { return fmt.Errorf("v2 fragment_basename is required") } switch d.Operation { case "create": - if d.PreviousFragmentSHA256 != "" || d.PreviousSidecarSHA256 != "" || d.Fragment == nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 != "" || d.Fragment == nil || d.Sidecar == nil { return fmt.Errorf("create document has inconsistent old/desired shape") } case "replace": - if d.PreviousFragmentSHA256 == "" || d.PreviousSidecarSHA256 == "" || d.Fragment == nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 == "" || d.Fragment == nil || d.Sidecar == nil { return fmt.Errorf("replace document has inconsistent old/desired shape") } case "remove": - if d.PreviousFragmentSHA256 == "" || d.PreviousSidecarSHA256 == "" || d.Fragment != nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 == "" || d.Fragment != nil || d.Sidecar == nil { return fmt.Errorf("remove document has inconsistent old/desired shape") } default: @@ -385,7 +390,7 @@ func (m *Manager) validateDocument(d Document) error { if err := validateParents(d.SidecarPath); err != nil { return err } - for _, want := range []string{d.PreviousFragmentSHA256, d.PreviousSidecarSHA256} { + for _, want := range []string{d.PreviousFragmentSHA256} { if want != "" && !validSHA(want) { return fmt.Errorf("previous artifact digest is invalid") } diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index 5f70be4..d804fba 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -16,7 +16,7 @@ import ( func TestApplyCreateResolveAndCleanup(t *testing.T) { m, dir := testManager(t) configDir := filepath.Join(dir, "conf") - fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") mustMkdir(t, filepath.Dir(fragmentPath)) if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { @@ -27,7 +27,7 @@ func TestApplyCreateResolveAndCleanup(t *testing.T) { } fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}` sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - doc := Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &fragment, Sidecar: &sidecar} + doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, Sidecar: &sidecar} applyDoc(t, m, doc, "task-a", "lease-a") result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()}) if err != nil || result.ExitCode != 0 { @@ -49,7 +49,7 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { m, dir := testManager(t) mustMkdir(t, filepath.Join(dir, "conf")) configDir := filepath.Join(dir, "conf") - fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { t.Fatal(err) @@ -62,7 +62,7 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { } desired := "new" desiredSidecar := "{}" - doc := Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, Fragment: &desired, Sidecar: &desiredSidecar} + doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &desired, Sidecar: &desiredSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") { t.Fatalf("unexpected error: %v", err) } @@ -81,7 +81,7 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { m, dir := testManager(t) mustMkdir(t, filepath.Join(dir, "conf")) configDir := filepath.Join(dir, "conf") - fragmentPath := filepath.Join(configDir, "lattice-linechain-chain.json") + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { t.Fatal(err) @@ -96,7 +96,7 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { } } newFragment, newSidecar := "new-fragment", "new-sidecar" - doc := Document{Version: 1, Operation: "replace", ConfigDir: configDir, FragmentPath: fragmentPath, SidecarPath: sidecarPath, PreviousFragmentSHA256: digest([]byte(oldFragment)), PreviousSidecarSHA256: digest([]byte(oldSidecar)), Fragment: &newFragment, Sidecar: &newSidecar} + doc := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") { t.Fatalf("unexpected error: %v", err) } @@ -107,7 +107,7 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { m, dir := testManager(t) mustMkdir(t, filepath.Join(dir, "conf")) - fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-chain.json") + fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") if err := m.ConfigureLayout(filepath.Join(dir, "conf"), sidecarPath); err != nil { t.Fatal(err) @@ -174,7 +174,8 @@ func TestValidateDocumentBindsOwnedPaths(t *testing.T) { if err := m.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil { t.Fatal(err) } - valid := BindDocument(Document{Version: 1, Operation: "create", ConfigDir: configDir, FragmentPath: filepath.Join(configDir, "lattice-linechain-a.json"), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar}) + basename := "lattice-linechain-0123456789abcdef0123.json" + valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: configDir, FragmentBasename: basename, FragmentPath: filepath.Join(configDir, basename), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar}) if err := m.validateDocument(valid); err != nil { t.Fatal(err) } @@ -227,7 +228,11 @@ func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) { } func applyDocErr(m *Manager, d Document, task, lease string) error { d = BindDocument(d) - b, _ := json.Marshal(d) + b, _ := json.Marshal(wireDocumentV2{ + Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, + PreviousFragmentSHA256: d.PreviousFragmentSHA256, Fragment: d.Fragment, Sidecar: d.Sidecar, + FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256, + }) return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease) } func assertFile(t *testing.T, p, want string) { From 7500f748b961e165e6931c1fbbc3435ba1d78bb0 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:07:52 -0700 Subject: [PATCH 21/43] Make runtime proof match the managed REALITY contract Exercise the exact VLESS+REALITY transport shape, make the crash boundary signal-driven, and model ordinary sidecar resync outside the E3 transaction so removal proves it tolerates unrelated metadata drift. Constraint: E3 v2 artifact authority binds only the fragment predecessor; ordinary metadata writers may change sidecar bytes independently Rejected: Manager-driven metadata resync | it hides the required independent-writer drift case Rejected: Polling published files as the crash trigger | it races a fast restart and can miss the intended journal phase Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep the crash marker inside the restart helper so pair_published is deterministic Tested: tagged helper compilation and go vet for cmd/lattice-agent Not-tested: Unsimulated real-binary lane pending worker-2 Manager contract repairs --- cmd/lattice-agent/linechain_e2e_test.go | 118 +++++++++++++++++++----- 1 file changed, 94 insertions(+), 24 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index d9a7722..d33105e 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -27,13 +28,14 @@ import ( ) const ( - linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN" - linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT" - linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR" - linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR" - linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT" - linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" - linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" + linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN" + linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT" + linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR" + linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR" + linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT" + linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" + linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" + linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER" ) // TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. The @@ -42,8 +44,20 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { bin := requireSingBoxE2EBinary(t) root := t.TempDir() origin := startEchoOrigin(t) - observer := startTCPObserver(t, origin.String()) aPort, bPort, clientPort := freePort(t), freePort(t), freePort(t) + observer := startTCPObserver(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(aPort))) + decoy := httptest.NewTLSServer(nil) + t.Cleanup(decoy.Close) + decoyAddress := strings.TrimPrefix(decoy.URL, "https://") + decoyHost, decoyPortText, err := net.SplitHostPort(decoyAddress) + if err != nil { + t.Fatal(err) + } + decoyPort, err := strconv.Atoi(decoyPortText) + if err != nil { + t.Fatal(err) + } + realityPrivate, realityPublic := generateRealityKeypair(t, bin) const ( uuidA = "11111111-1111-4111-8111-111111111111" uuidB = "22222222-2222-4222-8222-222222222222" @@ -60,11 +74,11 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { mustMkdir(t, clientDir) writeFile(t, filepath.Join(aDir, "config.json"), fmt.Sprintf(`{ "log":{"level":"error"}, - "inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q}]}], + "inbounds":[{"type":"vless","tag":"target-a","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q,"flow":"xtls-rprx-vision"}],"tls":{"enabled":true,"server_name":"e2e.lattice.invalid","reality":{"enabled":true,"handshake":{"server":%q,"server_port":%d},"private_key":%q,"short_id":["0123456789abcdef"]}}}], "outbounds":[{"type":"direct","tag":"direct"}], "route":{"rules":[{"inbound":["target-a"],"outbound":"direct"}]} } -`, aPort, uuidA)) +`, aPort, uuidA, decoyHost, decoyPort, realityPrivate)) writeFile(t, filepath.Join(bDir, "config.json"), fmt.Sprintf(`{ "log":{"level":"error"}, "inbounds":[{"type":"vless","tag":"source-b","listen":"127.0.0.1","listen_port":%d,"users":[{"uuid":%q}]}], @@ -81,6 +95,9 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { `, clientPort, bPort, uuidB)) startManagedSingBox(t, bin, root, "a", aDir, aPort) + if err := verifyManagedSingBox(root, "a", aPort); err != nil { + t.Fatalf("managed target A is inactive: %v", err) + } startManagedSingBox(t, bin, root, "b", bDir, bPort) startManagedSingBox(t, bin, root, "client", clientDir, clientPort) @@ -93,10 +110,10 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { fragmentPath := filepath.Join(bDir, basename) txnDir := filepath.Join(root, "txn") fragment := fmt.Sprintf(`{ - "outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":%q}], + "outbounds":[{"type":"vless","tag":"chain-to-a","server":"127.0.0.1","server_port":%d,"uuid":%q,"flow":"xtls-rprx-vision","tls":{"enabled":true,"server_name":"e2e.lattice.invalid","utls":{"enabled":true,"fingerprint":"chrome"},"reality":{"enabled":true,"public_key":%q,"short_id":"0123456789abcdef"}}}], "route":{"rules":[{"inbound":["source-b"],"outbound":"chain-to-a"}]} } -`, observer.Port(), uuidA) +`, observer.Port(), uuidA, realityPublic) sidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} `, lineUUIDB, lineUUIDA) @@ -106,9 +123,10 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { // Crash a real apply helper process group after it publishes both artifacts, // while this test (the agent/supervisor analogue) remains alive. Recovery // must restore and restart B before any inventory, traffic, or result callback. - crashDoc := bindE2EDocument("create", basename, "", "", &fragment, &sidecar) + crashDoc := bindE2EDocument("create", basename, "", &fragment, &sidecar) crashBytes := marshalE2EDocument(t, crashDoc) apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--") + crashMarker := filepath.Join(root, "restart-blocked") apply.Env = append(os.Environ(), linechainE2EBinEnv+"="+bin, linechainE2ERootEnv+"="+root, @@ -117,6 +135,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { linechainE2EBPortEnv+"="+strconv.Itoa(bPort), linechainE2ETaskEnv+"=crash-task", linechainE2ELeaseEnv+"=crash-lease", + linechainE2ECrashMarkerEnv+"="+crashMarker, ) apply.Stdin = bytes.NewReader(crashBytes) apply.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} @@ -125,11 +144,14 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { if err := apply.Start(); err != nil { t.Fatal(err) } + applyDone := make(chan error, 1) + go func() { applyDone <- apply.Wait() }() + waitForFileOrProcess(t, crashMarker, applyDone, &applyLog) waitForExactPair(t, fragmentPath, fragment, sidecarPath, sidecar) if err := syscall.Kill(-apply.Process.Pid, syscall.SIGKILL); err != nil { t.Fatalf("kill apply helper process group: %v", err) } - _ = apply.Wait() + <-applyDone order := []string{} if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { @@ -153,23 +175,26 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { } applyAndResolve(t, m, marshalE2EDocument(t, crashDoc), "create-task", "create-lease") + if err := verifyManagedSingBox(root, "a", aPort); err != nil { + t.Fatalf("managed target A became inactive after apply: %v", err) + } assertSOCKSEcho(t, clientPort, origin) if got := observer.Count(); got == 0 { t.Fatal("traffic did not traverse the B-to-A observer after apply") } assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) - // An ordinary metadata rewrite is a replace of the same fragment plus an - // augmented sidecar. The declared edge must survive the resync exactly. + // Simulate the ordinary independent metadata writer changing unrelated + // sidecar bytes. The declared edge remains intact, and remove must tolerate + // this non-E3 sidecar drift instead of applying a stale sidecar CAS. resyncedSidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","generated_by":"ordinary-resync","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} `, lineUUIDB, lineUUIDA) - replaceDoc := bindE2EDocument("replace", basename, digestText(fragment), digestText(sidecar), &fragment, &resyncedSidecar) - applyAndResolve(t, m, marshalE2EDocument(t, replaceDoc), "resync-task", "resync-lease") + writeFile(t, sidecarPath, resyncedSidecar) assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) removedSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]} ` - removeDoc := bindE2EDocument("remove", basename, digestText(fragment), digestText(resyncedSidecar), nil, &removedSidecar) + removeDoc := bindE2EDocument("remove", basename, digestText(fragment), nil, &removedSidecar) applyAndResolve(t, m, marshalE2EDocument(t, removeDoc), "remove-task", "remove-lease") if _, err := os.Stat(fragmentPath); !os.IsNotExist(err) { t.Fatalf("removed fragment still exists: %v", err) @@ -206,6 +231,14 @@ func TestLinechainE2ERestartHelper(t *testing.T) { if root == "" { return } + if marker := os.Getenv(linechainE2ECrashMarkerEnv); marker != "" { + if err := os.WriteFile(marker, []byte("pair_published\n"), 0o600); err != nil { + t.Fatal(err) + } + for { + time.Sleep(time.Hour) + } + } if err := restartManagedSingBox(os.Getenv(linechainE2EBinEnv), root, "b", os.Getenv(linechainE2EConfigEnv), mustEnvPort(linechainE2EBPortEnv)); err != nil { t.Fatal(err) } @@ -223,14 +256,11 @@ func TestLinechainE2EActiveHelper(t *testing.T) { type e2eDocument map[string]any -func bindE2EDocument(operation, basename, previousFragment, previousSidecar string, fragment, sidecar *string) e2eDocument { +func bindE2EDocument(operation, basename, previousFragment string, fragment, sidecar *string) e2eDocument { d := e2eDocument{"version": 2, "operation": operation, "fragment_basename": basename} if previousFragment != "" { d["previous_fragment_sha256"] = previousFragment } - if previousSidecar != "" { - d["previous_sidecar_sha256"] = previousSidecar - } if fragment != nil { d["fragment"] = *fragment } @@ -343,6 +373,27 @@ func findInventoryNode(t *testing.T, inv model.SingBoxInventory, name string) mo return model.SingBoxNode{} } +func generateRealityKeypair(t *testing.T, bin string) (string, string) { + t.Helper() + out, err := exec.Command(bin, "generate", "reality-keypair").CombinedOutput() + if err != nil { + t.Fatalf("generate reality keypair: %v: %s", err, out) + } + var privateKey, publicKey string + for _, line := range strings.Split(string(out), "\n") { + if value, ok := strings.CutPrefix(line, "PrivateKey: "); ok { + privateKey = strings.TrimSpace(value) + } + if value, ok := strings.CutPrefix(line, "PublicKey: "); ok { + publicKey = strings.TrimSpace(value) + } + } + if privateKey == "" || publicKey == "" { + t.Fatalf("unexpected reality keypair output: %s", out) + } + return privateKey, publicKey +} + func requireSingBoxE2EBinary(t *testing.T) string { t.Helper() bin := os.Getenv(linechainE2EBinEnv) @@ -542,6 +593,23 @@ func waitForPort(port int, timeout time.Duration) error { return fmt.Errorf("port %d did not become ready", port) } +func waitForFileOrProcess(t *testing.T, path string, done <-chan error, output *bytes.Buffer) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); err == nil { + return + } + select { + case err := <-done: + t.Fatalf("apply helper exited before crash marker: %v: %s", err, output.String()) + default: + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for helper signal %s: %s", path, output.String()) +} + func waitForExactPair(t *testing.T, fragmentPath, fragment, sidecarPath, sidecar string) { t.Helper() deadline := time.Now().Add(8 * time.Second) @@ -553,7 +621,9 @@ func waitForExactPair(t *testing.T, fragmentPath, fragment, sidecarPath, sidecar } time.Sleep(10 * time.Millisecond) } - t.Fatal("apply helper did not publish the exact pair before crash boundary") + f, ferr := os.ReadFile(fragmentPath) + s, serr := os.ReadFile(sidecarPath) + t.Fatalf("apply helper did not publish exact pair: fragment err=%v got=%q want=%q; sidecar err=%v got=%q want=%q", ferr, f, fragment, serr, s, sidecar) } func mustEnvPort(key string) int { From 6e0dd5426398db9d5fdcee609c5bd6f5c69bcf05 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 07:28:49 -0700 Subject: [PATCH 22/43] Cross-check E3 authority before recovery Canonicalize trusted runtime roots, reject trailing and legacy wire data, merge server-owned sidecar keys over ordinary metadata, and require exact task/lease/protocol agreement between bounded outbox and linechain snapshots before recovery can mutate either domain. Constraint: Completed E3 outbox entries may outlive cleaned linechain journals; leased E3 entries may not lack them Constraint: Ordinary sidecar top-level fields are outside E3 ownership and must survive create, replace, and remove Rejected: Byte-level sidecar predecessor CAS | independent metadata writers make it stale by design Confidence: high Scope-risk: moderate Reversibility: clean Directive: Run cross-contract validation before either generic outbox or linechain recovery Tested: go test ./internal/linechain ./internal/taskoutbox ./cmd/lattice-agent -count=1 Not-tested: Full/race/real-binary gates follow this commit --- cmd/lattice-agent/linechain_e2e_test.go | 20 +++++- cmd/lattice-agent/main.go | 48 +++++++++++++ internal/linechain/manager.go | 94 +++++++++++++++++++++++++ internal/linechain/manager_test.go | 83 ++++++++++++++++++++-- internal/taskoutbox/outbox.go | 9 +++ 5 files changed, 246 insertions(+), 8 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index d33105e..d531655 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -114,8 +114,9 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { "route":{"rules":[{"inbound":["source-b"],"outbound":"chain-to-a"}]} } `, observer.Port(), uuidA, realityPublic) - sidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} + sidecar := fmt.Sprintf(`{"inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}],"schema":"lattice.singbox-metadata.v2"} `, lineUUIDB, lineUUIDA) + sidecar = canonicalJSONString(t, sidecar) m := openE2EManager(t, bin, root, bDir, sidecarPath, txnDir, bPort) defer m.Close() @@ -187,12 +188,12 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { // Simulate the ordinary independent metadata writer changing unrelated // sidecar bytes. The declared edge remains intact, and remove must tolerate // this non-E3 sidecar drift instead of applying a stale sidecar CAS. - resyncedSidecar := fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","generated_by":"ordinary-resync","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}]} + resyncedSidecar := fmt.Sprintf(`{"generated_by":"ordinary-resync","inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}],"schema":"lattice.singbox-metadata.v2"} `, lineUUIDB, lineUUIDA) writeFile(t, sidecarPath, resyncedSidecar) assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) - removedSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]} + removedSidecar := `{"inbounds":[],"schema":"lattice.singbox-metadata.v2"} ` removeDoc := bindE2EDocument("remove", basename, digestText(fragment), nil, &removedSidecar) applyAndResolve(t, m, marshalE2EDocument(t, removeDoc), "remove-task", "remove-lease") @@ -290,6 +291,19 @@ func marshalE2EDocument(t *testing.T, d e2eDocument) []byte { return b } +func canonicalJSONString(t *testing.T, raw string) string { + t.Helper() + var value any + if err := json.Unmarshal([]byte(raw), &value); err != nil { + t.Fatal(err) + } + b, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(append(b, '\n')) +} + func openE2EManager(t *testing.T, bin, root, configDir, sidecarPath, txnDir string, bPort int) *linechain.Manager { t.Helper() for key, value := range map[string]string{ diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 131584f..af70f2c 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -1175,6 +1175,10 @@ type taskResultOutbox interface { Remove(taskoutbox.Entry) error } +type taskResultOutboxSnapshot interface { + Snapshot() ([]taskoutbox.Entry, error) +} + type leasedAgentTask struct { model.Task DurableResult bool `json:"durable_result"` @@ -1187,6 +1191,9 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag manager = managers[0] } if manager != nil { + if err := crossCheckLinechainAuthority(manager, outbox); err != nil { + return err + } if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil { return err } @@ -1324,6 +1331,47 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag return nil } +func crossCheckLinechainAuthority(manager *linechain.Manager, outbox taskResultOutbox) error { + typed, ok := outbox.(taskResultOutboxSnapshot) + if !ok { + return nil + } + entries, err := typed.Snapshot() + if err != nil { + return err + } + refs, err := manager.Snapshot() + if err != nil { + return err + } + journals := map[string]struct{}{} + for _, ref := range refs { + journals[ref.TaskID+"\x00"+ref.LeaseID] = struct{}{} + } + matchedE3 := map[string]struct{}{} + for _, entry := range entries { + key := entry.Task.ID + "\x00" + entry.Task.LeaseID + _, hasJournal := journals[key] + if hasJournal && entry.DurableProtocol != "linechain-e3-v1" { + return fmt.Errorf("linechain journal %s has mismatched outbox protocol", entry.Task.ID) + } + if hasJournal && entry.DurableProtocol == "linechain-e3-v1" { + matchedE3[key] = struct{}{} + } + if entry.State == "leased" && entry.DurableProtocol == "linechain-e3-v1" { + if !hasJournal { + return fmt.Errorf("leased E3 outbox %s lacks exact linechain journal", entry.Task.ID) + } + } + } + for key := range journals { + if _, ok := matchedE3[key]; !ok { + return fmt.Errorf("linechain journal lacks exact leased E3 outbox: %q", key) + } + } + return nil +} + func validateDurablePair(task leasedAgentTask, manager *linechain.Manager, ready bool) error { if !task.DurableResult { if task.DurableProtocol != "" { diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index ec9216a..c25cf74 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -113,6 +113,22 @@ func (m *Manager) ConfigureLayout(configDir, sidecarPath string) error { if !filepath.IsAbs(configDir) || !filepath.IsAbs(sidecarPath) { return fmt.Errorf("linechain layout paths must be absolute") } + for label, root := range map[string]string{"config": configDir, "sidecar": filepath.Dir(sidecarPath)} { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) || info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("linechain %s root is not a trusted private directory", label) + } + } + resolvedConfig, err := filepath.EvalSymlinks(configDir) + if err != nil { + return fmt.Errorf("resolve linechain config directory: %w", err) + } + resolvedSidecarParent, err := filepath.EvalSymlinks(filepath.Dir(sidecarPath)) + if err != nil { + return fmt.Errorf("resolve linechain sidecar directory: %w", err) + } + configDir = filepath.Clean(resolvedConfig) + sidecarPath = filepath.Join(filepath.Clean(resolvedSidecarParent), filepath.Base(sidecarPath)) if err := validateParents(filepath.Join(configDir, "placeholder")); err != nil { return err } @@ -264,6 +280,12 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := dec.Decode(&wire); err != nil { return fmt.Errorf("decode linechain document: %w", err) } + var trailing any + if err := dec.Decode(&trailing); err == nil { + return fmt.Errorf("decode linechain document: unexpected trailing data") + } else if !errors.Is(err, io.EOF) { + return fmt.Errorf("decode linechain document trailing data: %w", err) + } if wire.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", wire.Version) } @@ -279,6 +301,12 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := m.validateDocument(d); err != nil { return err } + mergedSidecar, err := mergeSidecar(sidecarCurrent(m.sidecarPath), d.Sidecar) + if err != nil { + return err + } + d.Sidecar = mergedSidecar + d = BindDocument(d) path := m.journalPath(taskID, leaseID) if _, err := os.Lstat(path); err == nil { return m.recoverOne(ctx, path, nil, "") @@ -328,6 +356,72 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string return writeJSON(path, j) } +func sidecarCurrent(path string) *string { + b, err := os.ReadFile(path) + if err != nil { + return nil + } + s := string(b) + return &s +} + +func mergeSidecar(current, desired *string) (*string, error) { + if desired == nil { + return nil, nil + } + merged := map[string]any{} + if current != nil { + if err := json.Unmarshal([]byte(*current), &merged); err != nil { + return nil, fmt.Errorf("decode current semantic sidecar: %w", err) + } + } + want := map[string]any{} + if err := json.Unmarshal([]byte(*desired), &want); err != nil { + return nil, fmt.Errorf("decode desired semantic sidecar: %w", err) + } + if want["schema"] != "lattice.singbox-metadata.v2" { + return nil, fmt.Errorf("desired semantic sidecar has unsupported schema") + } + if _, ok := want["inbounds"].([]any); !ok { + return nil, fmt.Errorf("desired semantic sidecar inbounds must be an array") + } + for key, value := range want { + merged[key] = value + } + b, err := json.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("encode semantic sidecar: %w", err) + } + b = append(b, '\n') + s := string(b) + return &s, nil +} + +// JournalRef is the bounded identity needed to cross-check outbox authority. +type JournalRef struct{ TaskID, LeaseID string } + +func (m *Manager) Snapshot() ([]JournalRef, error) { + entries, err := os.ReadDir(m.dir) + if err != nil { + return nil, err + } + refs := make([]JournalRef, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + if len(refs) >= 1024 { + return nil, fmt.Errorf("linechain journal capacity exceeded") + } + j, err := readJournal(filepath.Join(m.dir, entry.Name())) + if err != nil { + return nil, err + } + refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID}) + } + return refs, nil +} + func (m *Manager) validateDocument(d Document) error { if d.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", d.Version) diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index d804fba..ab9b8a6 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -61,7 +61,7 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { t.Fatal(err) } desired := "new" - desiredSidecar := "{}" + desiredSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &desired, Sidecar: &desiredSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") { t.Fatalf("unexpected error: %v", err) @@ -89,13 +89,13 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { if err := m.ConfigureCommands("false", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - oldFragment, oldSidecar := "old-fragment", "old-sidecar" + oldFragment, oldSidecar := "old-fragment", `{"ordinary":"old"}` for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} { if err := os.WriteFile(path, []byte(data), 0o600); err != nil { t.Fatal(err) } } - newFragment, newSidecar := "new-fragment", "new-sidecar" + newFragment, newSidecar := "new-fragment", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` doc := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") { t.Fatalf("unexpected error: %v", err) @@ -170,12 +170,12 @@ func TestValidateDocumentBindsOwnedPaths(t *testing.T) { configDir := filepath.Join(root, "conf") mustMkdir(t, configDir) fragment := "{}" - sidecar := "{}" + sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` if err := m.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil { t.Fatal(err) } basename := "lattice-linechain-0123456789abcdef0123.json" - valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: configDir, FragmentBasename: basename, FragmentPath: filepath.Join(configDir, basename), SidecarPath: filepath.Join(root, "lattice-metadata.json"), Fragment: &fragment, Sidecar: &sidecar}) + valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: m.configDir, FragmentBasename: basename, FragmentPath: filepath.Join(m.configDir, basename), SidecarPath: m.sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) if err := m.validateDocument(valid); err != nil { t.Fatal(err) } @@ -204,6 +204,79 @@ func TestResolvePreflightFailureWithoutJournal(t *testing.T) { } } +func TestApplyRejectsTrailingAndLegacyWireFields(t *testing.T) { + m, dir := testManager(t) + configDir := filepath.Join(dir, "conf") + mustMkdir(t, configDir) + if err := m.ConfigureLayout(configDir, filepath.Join(dir, "lattice-metadata.json")); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := BindDocument(Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) + base, _ := json.Marshal(wireDocumentV2{Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, Fragment: d.Fragment, Sidecar: d.Sidecar, FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256}) + for name, raw := range map[string]string{ + "trailing": string(base) + ` {}`, + "legacy path": strings.TrimSuffix(string(base), "}") + `,"config_dir":"/tmp"}`, + "legacy sidecar digest": strings.TrimSuffix(string(base), "}") + `,"previous_sidecar_sha256":"` + strings.Repeat("0", 64) + `"}`, + } { + t.Run(name, func(t *testing.T) { + if err := m.Apply(context.Background(), strings.NewReader(raw), "task-"+name, "lease"); err == nil { + t.Fatal("invalid wire accepted") + } + }) + } +} + +func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { + m, dir := testManager(t) + configDir := filepath.Join(dir, "conf") + mustMkdir(t, configDir) + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err := os.WriteFile(sidecarPath, []byte(`{"ordinary":{"owner":"sb"},"inbounds":[{"tag":"old"}]}`), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"managed"}]}` + d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar} + applyDoc(t, m, d, "task-overlay", "lease-overlay") + b, err := os.ReadFile(sidecarPath) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + if got["ordinary"] == nil || got["schema"] != "lattice.singbox-metadata.v2" { + t.Fatalf("overlay lost fields: %s", b) + } +} + +func TestConfigureLayoutRejectsSymlinkRoot(t *testing.T) { + m, dir := testManager(t) + realDir := filepath.Join(dir, "real") + mustMkdir(t, realDir) + link := filepath.Join(dir, "link") + if err := os.Symlink(realDir, link); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(link, filepath.Join(dir, "meta.json")); err == nil { + t.Fatal("symlink config root accepted") + } + badParent := filepath.Join(dir, "bad-parent") + if err := os.Symlink(realDir, badParent); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(realDir, filepath.Join(badParent, "meta.json")); err == nil { + t.Fatal("symlink sidecar root accepted") + } +} + func testManager(t *testing.T) (*Manager, string) { t.Helper() dir := filepath.Join(t.TempDir(), "txn") diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index ec84704..443f3bf 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -308,6 +308,15 @@ func (s *Store) Pending() ([]Entry, error) { return pending, nil } +// Snapshot returns a bounded copy of every leased and completed entry. +func (s *Store) Snapshot() ([]Entry, error) { + entries, err := s.readAll() + if err != nil { + return nil, err + } + return append([]Entry(nil), entries...), nil +} + // Remove atomically unlinks an acknowledged outbox entry and syncs the // directory so the acknowledgement survives a crash. func (s *Store) Remove(entry Entry) error { From 341ced24c648986257e519f6bc0f56f88c86b865 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 08:04:08 -0700 Subject: [PATCH 23/43] Make E3 recovery authority survive hostile local state The managed linechain transaction now binds server-approved task scripts to canonical local artifacts, validates complete journal authority across the outbox handoff, preserves semantic sidecar ownership, and proves runtime state before recording terminal results. Trusted task execution no longer exposes linechain paths to ordinary or NetGuard tasks, while nofollow bounded reads and fault-matrix coverage close recovery and rollback ambiguity. Constraint: E3 recovery must suppress observable networking until exact outbox, journal, artifact, and runtime authority agree Constraint: Existing ordinary sidecar fields remain locally owned and survive managed overlay updates Rejected: Trust task and lease identifiers alone | they do not bind the approved script or desired artifacts Rejected: Include command output in durable failures | checker output may contain credentials Confidence: high Scope-risk: moderate Reversibility: clean Directive: Do not bypass RunLinechain protocol selection or weaken full-journal authority capture before recovery Tested: go test ./... -count=1; go vet ./...; go test -race -cover ./... -count=1; exact unsimulated sing-box 1.13.18 E2E; release workflow and install integrity scripts; version and compatibility output; secret scan Not-tested: GitHub CI on this unpushed commit --- cmd/lattice-agent/linechain_e2e_test.go | 113 +++- cmd/lattice-agent/main.go | 87 ++- cmd/lattice-agent/main_test.go | 283 ++++++++- go.mod | 2 +- internal/linechain/manager.go | 378 ++++++++--- internal/linechain/manager_test.go | 723 +++++++++++++++++++++- internal/linechain/read_nofollow_other.go | 26 + internal/linechain/read_nofollow_unix.go | 18 + internal/taskexec/taskexec.go | 19 +- internal/taskexec/taskexec_test.go | 27 + scripts/test-linechain-e2e.sh | 34 +- 11 files changed, 1563 insertions(+), 147 deletions(-) create mode 100644 internal/linechain/read_nofollow_other.go create mode 100644 internal/linechain/read_nofollow_unix.go diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index d531655..0c2fa74 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -38,11 +39,22 @@ const ( linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER" ) +var managedE2EProcesses = struct { + sync.Mutex + done map[int]<-chan error +}{done: make(map[int]<-chan error)} + // TestLinechainRealSingBoxE2E is invoked by scripts/test-linechain-e2e.sh. The // script makes the real 1.13.x binary mandatory, so this test never skips. func TestLinechainRealSingBoxE2E(t *testing.T) { bin := requireSingBoxE2EBinary(t) - root := t.TempDir() + root := os.Getenv(linechainE2ERootEnv) + if !filepath.IsAbs(root) { + t.Fatalf("%s must be an absolute test root", linechainE2ERootEnv) + } + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } origin := startEchoOrigin(t) aPort, bPort, clientPort := freePort(t), freePort(t), freePort(t) observer := startTCPObserver(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(aPort))) @@ -126,7 +138,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { // must restore and restart B before any inventory, traffic, or result callback. crashDoc := bindE2EDocument("create", basename, "", &fragment, &sidecar) crashBytes := marshalE2EDocument(t, crashDoc) - apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--") + apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--", root) crashMarker := filepath.Join(root, "restart-blocked") apply.Env = append(os.Environ(), linechainE2EBinEnv+"="+bin, @@ -136,6 +148,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { linechainE2EBPortEnv+"="+strconv.Itoa(bPort), linechainE2ETaskEnv+"=crash-task", linechainE2ELeaseEnv+"=crash-lease", + "LATTICE_LINECHAIN_TASK_SCRIPT_SHA256="+digestText("e2e-helper-script"), linechainE2ECrashMarkerEnv+"="+crashMarker, ) apply.Stdin = bytes.NewReader(crashBytes) @@ -147,12 +160,22 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { } applyDone := make(chan error, 1) go func() { applyDone <- apply.Wait() }() + var cleanupApply sync.Once + stopApply := func() { + cleanupApply.Do(func() { + _ = syscall.Kill(-apply.Process.Pid, syscall.SIGKILL) + select { + case <-applyDone: + case <-time.After(5 * time.Second): + t.Errorf("apply helper did not exit after process-group kill") + } + killMarkerProcess(t, crashMarker) + }) + } + t.Cleanup(stopApply) waitForFileOrProcess(t, crashMarker, applyDone, &applyLog) waitForExactPair(t, fragmentPath, fragment, sidecarPath, sidecar) - if err := syscall.Kill(-apply.Process.Pid, syscall.SIGKILL); err != nil { - t.Fatalf("kill apply helper process group: %v", err) - } - <-applyDone + stopApply() order := []string{} if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { @@ -220,7 +243,7 @@ func TestLinechainE2EApplyHelper(t *testing.T) { } defer m.Close() configureE2EManager(t, m) - if err := m.Apply(context.Background(), os.Stdin, os.Getenv(linechainE2ETaskEnv), os.Getenv(linechainE2ELeaseEnv)); err != nil { + if err := m.Apply(context.Background(), os.Stdin, os.Getenv(linechainE2ETaskEnv), os.Getenv(linechainE2ELeaseEnv), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil { t.Fatal(err) } } @@ -233,7 +256,7 @@ func TestLinechainE2ERestartHelper(t *testing.T) { return } if marker := os.Getenv(linechainE2ECrashMarkerEnv); marker != "" { - if err := os.WriteFile(marker, []byte("pair_published\n"), 0o600); err != nil { + if err := os.WriteFile(marker, []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600); err != nil { t.Fatal(err) } for { @@ -328,8 +351,9 @@ func configureE2EManager(t *testing.T, m *linechain.Manager) { if err := m.ConfigureLayout(os.Getenv(linechainE2EConfigEnv), os.Getenv(linechainE2ESidecarEnv)); err != nil { t.Fatal(err) } - restart := []string{os.Args[0], "-test.run=^TestLinechainE2ERestartHelper$", "--"} - verify := []string{os.Args[0], "-test.run=^TestLinechainE2EActiveHelper$", "--"} + root := os.Getenv(linechainE2ERootEnv) + restart := []string{os.Args[0], "-test.run=^TestLinechainE2ERestartHelper$", "--", root} + verify := []string{os.Args[0], "-test.run=^TestLinechainE2EActiveHelper$", "--", root} if err := m.ConfigureCommands(os.Getenv(linechainE2EBinEnv), restart, verify); err != nil { t.Fatal(err) } @@ -337,7 +361,7 @@ func configureE2EManager(t *testing.T, m *linechain.Manager) { func applyAndResolve(t *testing.T, m *linechain.Manager, document []byte, taskID, leaseID string) { t.Helper() - if err := m.Apply(context.Background(), bytes.NewReader(document), taskID, leaseID); err != nil { + if err := m.Apply(context.Background(), bytes.NewReader(document), taskID, leaseID, digestText("e2e-direct:"+taskID)); err != nil { t.Fatal(err) } result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0}) @@ -536,6 +560,11 @@ func restartManagedSingBox(bin, root, name, configDir string, port int) error { _ = logFile.Close() return err } + done := make(chan error, 1) + managedE2EProcesses.Lock() + managedE2EProcesses.done[cmd.Process.Pid] = done + managedE2EProcesses.Unlock() + go func() { done <- cmd.Wait() }() _ = logFile.Close() if err := os.WriteFile(filepath.Join(root, name+".pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) @@ -565,9 +594,16 @@ func killManagedSingBox(root, name string) error { if err := syscall.Kill(pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH { return err } + managedE2EProcesses.Lock() + done := managedE2EProcesses.done[pid] + managedE2EProcesses.Unlock() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { - if err := syscall.Kill(pid, 0); err == syscall.ESRCH { + if managedProcessExited(pid, done) { + managedE2EProcesses.Lock() + delete(managedE2EProcesses.done, pid) + managedE2EProcesses.Unlock() + _ = os.Remove(pidPath) return nil } time.Sleep(20 * time.Millisecond) @@ -575,7 +611,58 @@ func killManagedSingBox(root, name string) error { if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { return err } - return nil + killDeadline := time.Now().Add(2 * time.Second) + for time.Now().Before(killDeadline) { + if managedProcessExited(pid, done) { + managedE2EProcesses.Lock() + delete(managedE2EProcesses.done, pid) + managedE2EProcesses.Unlock() + _ = os.Remove(pidPath) + return nil + } + time.Sleep(20 * time.Millisecond) + } + return fmt.Errorf("%s process %d remained after SIGKILL", name, pid) +} + +func managedProcessExited(pid int, done <-chan error) bool { + if done != nil { + select { + case <-done: + return true + default: + return false + } + } + return syscall.Kill(pid, 0) == syscall.ESRCH +} + +func killMarkerProcess(t *testing.T, marker string) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + raw, err := os.ReadFile(marker) + if os.IsNotExist(err) { + return + } + if err != nil { + t.Errorf("read helper marker: %v", err) + return + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || pid <= 1 { + t.Errorf("invalid helper marker %q", raw) + return + } + _ = syscall.Kill(-pid, syscall.SIGKILL) + _ = syscall.Kill(pid, syscall.SIGKILL) + if err := syscall.Kill(pid, 0); err == syscall.ESRCH { + _ = os.Remove(marker) + return + } + time.Sleep(20 * time.Millisecond) + } + t.Errorf("restart helper from %s remained after cleanup", marker) } func verifyManagedSingBox(root, name string, port int) error { diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index af70f2c..9575209 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "flag" @@ -294,7 +295,7 @@ func main() { if err := manager.ConfigureLayout(os.Getenv("LATTICE_LINECHAIN_CONFIG_DIR"), os.Getenv("LATTICE_LINECHAIN_SIDECAR_PATH")); err != nil { log.Fatalf("linechain local layout failed: %v", err) } - if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID")); err != nil { + if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID"), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil { log.Fatalf("linechain apply failed: %v", err) } return @@ -1191,9 +1192,6 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag manager = managers[0] } if manager != nil { - if err := crossCheckLinechainAuthority(manager, outbox); err != nil { - return err - } if err := requireLinechainRecovered(context.Background(), manager, outbox, cfg.NodeID); err != nil { return err } @@ -1282,7 +1280,18 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag continue } debugf(cfg, "task start: id=%s interpreter=%s timeout=%ds", task.ID, task.Interpreter, task.TimeoutSec) - result := runner.Run(task) + var result model.TaskResult + if isLinechainTask(leased) { + typed, ok := runner.(interface { + RunLinechain(model.Task) model.TaskResult + }) + if !ok { + return fmt.Errorf("linechain task runner does not expose trusted E3 execution") + } + result = typed.RunLinechain(task) + } else { + result = runner.Run(task) + } result.NodeID = cfg.NodeID if manager != nil && isLinechainTask(leased) { result, err = manager.ResolveAfterRun(context.Background(), task, result) @@ -1332,44 +1341,84 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag } func crossCheckLinechainAuthority(manager *linechain.Manager, outbox taskResultOutbox) error { + _, err := captureLinechainAuthority(manager, outbox) + return err +} + +func captureLinechainAuthority(manager *linechain.Manager, outbox taskResultOutbox) (map[string]linechain.RecoveryAuthority, error) { typed, ok := outbox.(taskResultOutboxSnapshot) if !ok { - return nil + return nil, nil } entries, err := typed.Snapshot() if err != nil { - return err + return nil, err } refs, err := manager.Snapshot() if err != nil { - return err + return nil, err } - journals := map[string]struct{}{} + journals := map[string]linechain.JournalRef{} for _, ref := range refs { - journals[ref.TaskID+"\x00"+ref.LeaseID] = struct{}{} + journals[ref.TaskID+"\x00"+ref.LeaseID] = ref } matchedE3 := map[string]struct{}{} for _, entry := range entries { key := entry.Task.ID + "\x00" + entry.Task.LeaseID - _, hasJournal := journals[key] + ref, hasJournal := journals[key] if hasJournal && entry.DurableProtocol != "linechain-e3-v1" { - return fmt.Errorf("linechain journal %s has mismatched outbox protocol", entry.Task.ID) + return nil, fmt.Errorf("linechain journal %s has mismatched outbox protocol", entry.Task.ID) } if hasJournal && entry.DurableProtocol == "linechain-e3-v1" { + if ref.TaskScriptSHA != linechainTaskScriptSHA(entry.Task.Script) { + return nil, fmt.Errorf("linechain journal %s does not match the exact outbox task script", entry.Task.ID) + } matchedE3[key] = struct{}{} } if entry.State == "leased" && entry.DurableProtocol == "linechain-e3-v1" { if !hasJournal { - return fmt.Errorf("leased E3 outbox %s lacks exact linechain journal", entry.Task.ID) + return nil, fmt.Errorf("leased E3 outbox %s lacks exact linechain journal", entry.Task.ID) + } + } + if entry.State == "completed" && entry.DurableProtocol == "linechain-e3-v1" && hasJournal { + if !ref.Terminal || entry.Result == nil || ref.Result == nil || !sameTaskResult(*entry.Result, *ref.Result) { + return nil, fmt.Errorf("completed E3 outbox %s conflicts with linechain terminal result", entry.Task.ID) } } } for key := range journals { if _, ok := matchedE3[key]; !ok { - return fmt.Errorf("linechain journal lacks exact leased E3 outbox: %q", key) + return nil, fmt.Errorf("linechain journal lacks exact leased E3 outbox: %q", key) } } - return nil + authority := make(map[string]linechain.RecoveryAuthority, len(journals)) + for key, ref := range journals { + authority[key] = linechain.RecoveryAuthority{TaskScriptSHA: ref.TaskScriptSHA, CombinedSHA256: ref.CombinedSHA256, Phase: ref.Phase, ResultSHA256: taskResultPointerSHA(ref.Result), JournalSHA256: ref.JournalSHA256} + } + return authority, nil +} + +func sameTaskResult(a, b model.TaskResult) bool { + aBytes, aErr := json.Marshal(a) + bBytes, bErr := json.Marshal(b) + return aErr == nil && bErr == nil && bytes.Equal(aBytes, bBytes) +} + +func linechainTaskScriptSHA(script string) string { + sum := sha256.Sum256([]byte(script)) + return hex.EncodeToString(sum[:]) +} + +func taskResultPointerSHA(result *model.TaskResult) string { + if result == nil { + return "" + } + b, err := json.Marshal(result) + if err != nil { + return "" + } + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) } func validateDurablePair(task leasedAgentTask, manager *linechain.Manager, ready bool) error { @@ -1393,7 +1442,11 @@ func isLinechainTask(task leasedAgentTask) bool { } func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { - return manager.RequireRecovered(ctx, func(result model.TaskResult) error { + authority, err := captureLinechainAuthority(manager, outbox) + if err != nil { + return err + } + return manager.RequireRecoveredAuthorized(ctx, func(result model.TaskResult) error { committed, err := outbox.Complete(result) if err == nil { return nil @@ -1404,7 +1457,7 @@ func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, // An exact completed outbox may remain after a crash before transaction // cleanup. Complete implementations treat that replay idempotently. return err - }, nodeID) + }, nodeID, authority) } func flushTaskResults(cfg agentConfig, outbox taskResultOutbox) error { diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index d3f682e..aa0f0c1 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/LatticeNet/lattice-node-agent/internal/linechain" + "github.com/LatticeNet/lattice-node-agent/internal/taskexec" "github.com/LatticeNet/lattice-node-agent/internal/taskoutbox" "github.com/LatticeNet/lattice-sdk/model" ) @@ -310,19 +311,28 @@ func TestRunTasksRetainsResultAcrossTransientServerFailure(t *testing.T) { } type linechainTaskRunner struct { - manager *linechain.Manager - doc []byte - calls int - afterApply func() + manager *linechain.Manager + doc []byte + linechainTaskID string + calls int + callsByID map[string]int + afterApply func() } func (r *linechainTaskRunner) Run(task model.Task) model.TaskResult { r.calls++ - err := r.manager.Apply(context.Background(), bytes.NewReader(r.doc), task.ID, task.LeaseID) + if r.callsByID == nil { + r.callsByID = map[string]int{} + } + r.callsByID[task.ID]++ + result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, StartedAt: time.Now().UTC(), FinishedAt: time.Now().UTC()} + if task.ID != r.linechainTaskID { + return result + } + err := r.manager.Apply(context.Background(), bytes.NewReader(r.doc), task.ID, task.LeaseID, linechainTaskScriptSHA(task.Script)) if err == nil && r.afterApply != nil { r.afterApply() } - result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, StartedAt: time.Now().UTC(), FinishedAt: time.Now().UTC()} if err != nil { result.ExitCode = -1 result.Error = err.Error() @@ -330,6 +340,8 @@ func (r *linechainTaskRunner) Run(task model.Task) model.TaskResult { return result } +func (r *linechainTaskRunner) RunLinechain(task model.Task) model.TaskResult { return r.Run(task) } + func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { oldClient := httpClient t.Cleanup(func() { httpClient = oldClient }) @@ -368,7 +380,9 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { t.Fatal(err) } task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: "# lattice-linechain-e3-v1\n\"$LATTICE_LINECHAIN_TXN_DIR\"=1 \"$LATTICE_AGENT_BIN\" -linechain-apply", TimeoutSec: 10, OutputLimit: 1024} - runner := &linechainTaskRunner{manager: manager, doc: doc, afterApply: func() { + genericTask := model.Task{ID: "task-generic", LeaseID: "lease-generic", Interpreter: "sh", Script: "echo generic"} + netguardTask := model.Task{ID: "task-netguard", LeaseID: "lease-netguard", Interpreter: "sh", Script: "echo netguard"} + runner := &linechainTaskRunner{manager: manager, doc: doc, linechainTaskID: task.ID, afterApply: func() { manager.ConfigureCleanupForTest(func(path string) error { if strings.HasSuffix(path, ".json") { return errors.New("injected cleanup failure") @@ -384,7 +398,11 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { case "/api/agent/tasks": fetches++ if fetches == 1 { - b, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v1"}}) + b, _ := json.Marshal([]leasedAgentTask{ + {Task: genericTask}, + {Task: netguardTask, DurableResult: true, DurableProtocol: "netguard-v1"}, + {Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v1"}, + }) return testResponse(http.StatusOK, string(b)), nil } return testResponse(http.StatusOK, "[]"), nil @@ -413,8 +431,11 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { if err := runTasks(cfg, runner, outbox, manager); err != nil { t.Fatal(err) } - if runner.calls != 1 || posts != 2 || len(posted) != 2 || !reflect.DeepEqual(posted[0], posted[1]) { - t.Fatalf("runner calls=%d posts=%d", runner.calls, posts) + if runner.calls != 3 || runner.callsByID[genericTask.ID] != 1 || runner.callsByID[netguardTask.ID] != 1 || runner.callsByID[task.ID] != 1 { + t.Fatalf("mixed task execution was replayed or skipped: total=%d byID=%v", runner.calls, runner.callsByID) + } + if posts != 4 || len(posted) != 4 || posted[0].TaskID != genericTask.ID || posted[1].TaskID != netguardTask.ID || !reflect.DeepEqual(posted[2], posted[3]) { + t.Fatalf("mixed result posts=%d results=%+v", posts, posted) } if entries, err := os.ReadDir(txnDir); err != nil { t.Fatal(err) @@ -427,6 +448,230 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { } } +func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { + newPair := func(t *testing.T) (*linechain.Manager, *taskoutbox.Store) { + t.Helper() + root := t.TempDir() + manager, err := linechain.Open(filepath.Join(root, "txn")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = manager.Close() }) + outbox, err := taskoutbox.Open(filepath.Join(root, "outbox")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = outbox.Close() }) + return manager, outbox + } + task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: "true"} + + t.Run("leased E3 without journal blocks require gate", func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + if err := requireLinechainRecovered(context.Background(), manager, outbox, "node-a"); err == nil || !strings.Contains(err.Error(), "lacks exact linechain journal") { + t.Fatalf("require gate error = %v", err) + } + }) + + t.Run("completed E3 without journal is allowed", func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + if _, err := outbox.Complete(model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := crossCheckLinechainAuthority(manager, outbox); err != nil { + t.Fatal(err) + } + }) + + t.Run("journal without exact outbox is blocked", func(t *testing.T) { + manager, outbox := newPair(t) + createLinechainJournal(t, manager, task) + if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "lacks exact") { + t.Fatalf("cross-check error = %v", err) + } + }) + + t.Run("journal with wrong protocol is blocked", func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "netguard-v1"); err != nil { + t.Fatal(err) + } + createLinechainJournal(t, manager, task) + if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "mismatched outbox protocol") { + t.Fatalf("cross-check error = %v", err) + } + }) + + t.Run("same IDs with different script are blocked", func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + other := task + other.Script = "different approved artifact" + createLinechainJournal(t, manager, other) + if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "exact outbox task script") { + t.Fatalf("error=%v", err) + } + }) + + t.Run("completed E3 requires exact terminal result", func(t *testing.T) { + for _, mismatch := range []bool{false, true} { + t.Run(map[bool]string{false: "matching", true: "mismatched"}[mismatch], func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + createLinechainJournal(t, manager, task) + result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()} + if _, err := manager.ResolveAfterRun(context.Background(), task, result); err != nil { + t.Fatal(err) + } + outboxResult := result + if mismatch { + outboxResult.ExitCode = -1 + outboxResult.Error = "different" + } + if _, err := outbox.Complete(outboxResult); err != nil { + t.Fatal(err) + } + err := crossCheckLinechainAuthority(manager, outbox) + if mismatch && (err == nil || !strings.Contains(err.Error(), "conflicts")) { + t.Fatalf("error=%v", err) + } + if !mismatch && err != nil { + t.Fatal(err) + } + }) + } + }) + + t.Run("completed E3 conflicts with nonterminal journal", func(t *testing.T) { + manager, outbox := newPair(t) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + createLinechainJournal(t, manager, task) + if _, err := outbox.Complete(model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "conflicts") { + t.Fatalf("error=%v", err) + } + }) +} + +func createLinechainJournal(t *testing.T, manager *linechain.Manager, task model.Task) { + t.Helper() + root := t.TempDir() + configDir := filepath.Join(root, "conf") + if err := os.Mkdir(configDir, 0o700); err != nil { + t.Fatal(err) + } + if err := manager.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil { + t.Fatal(err) + } + if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) + raw, err := json.Marshal(map[string]any{ + "version": d.Version, "operation": d.Operation, "fragment_basename": d.FragmentBasename, + "fragment": d.Fragment, "sidecar": d.Sidecar, "fragment_sha256": d.FragmentSHA256, + "sidecar_sha256": d.SidecarSHA256, "combined_sha256": d.CombinedSHA256, + }) + if err != nil { + t.Fatal(err) + } + if err := manager.Apply(context.Background(), bytes.NewReader(raw), task.ID, task.LeaseID, linechainTaskScriptSHA(task.Script)); err != nil { + t.Fatal(err) + } +} + +func TestTaskexecRunsRealLinechainHelperShell(t *testing.T) { + root := t.TempDir() + txnDir := filepath.Join(root, "txn") + configDir := filepath.Join(root, "conf") + sidecarPath := filepath.Join(root, "lattice-metadata.json") + workdir := filepath.Join(root, "work") + for _, dir := range []string{configDir, workdir} { + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + } + manager, err := linechain.Open(txnDir) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + if err := manager.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{"outbounds":[]}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) + raw, err := json.Marshal(map[string]any{ + "version": d.Version, "operation": d.Operation, "fragment_basename": d.FragmentBasename, + "fragment": d.Fragment, "sidecar": d.Sidecar, "fragment_sha256": d.FragmentSHA256, + "sidecar_sha256": d.SidecarSHA256, "combined_sha256": d.CombinedSHA256, + }) + if err != nil { + t.Fatal(err) + } + testBinary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + task := model.Task{ + ID: "task-real-shell", LeaseID: "lease-real-shell", Interpreter: "sh", + Script: "printf '%s' '" + string(raw) + "' | LATTICE_TEST_LINECHAIN_HELPER=1 \"$LATTICE_AGENT_BIN\" -test.run=TestTaskexecLinechainHelperProcess", + TimeoutSec: 20, OutputLimit: 4096, + } + runner := taskexec.Runner{ + AllowExec: true, AllowRoot: true, WorkdirRoot: workdir, AgentBinary: testBinary, + LinechainTxnDir: txnDir, LinechainConfigDir: configDir, LinechainSidecarPath: sidecarPath, + } + result := runner.RunLinechain(task) + if result.ExitCode != 0 || result.Error != "" { + t.Fatalf("real taskexec helper failed: %+v stderr=%s", result, result.Stderr) + } + result, err = manager.ResolveAfterRun(context.Background(), task, result) + if err != nil || result.ExitCode != 0 { + t.Fatalf("resolve real taskexec helper: result=%+v err=%v", result, err) + } + if err := manager.Cleanup(task.ID, task.LeaseID); err != nil { + t.Fatal(err) + } +} + +func TestTaskexecLinechainHelperProcess(t *testing.T) { + if os.Getenv("LATTICE_TEST_LINECHAIN_HELPER") != "1" { + return + } + manager, err := linechain.OpenHelper(os.Getenv("LATTICE_LINECHAIN_TXN_DIR")) + if err != nil { + t.Fatal(err) + } + if err := manager.ConfigureLayout(os.Getenv("LATTICE_LINECHAIN_CONFIG_DIR"), os.Getenv("LATTICE_LINECHAIN_SIDECAR_PATH")); err != nil { + t.Fatal(err) + } + if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + if err := manager.Apply(context.Background(), os.Stdin, os.Getenv("LATTICE_TASK_ID"), os.Getenv("LATTICE_TASK_LEASE_ID"), os.Getenv("LATTICE_LINECHAIN_TASK_SCRIPT_SHA256")); err != nil { + t.Fatal(err) + } +} + func TestRunTasksRestartFlushesCompletedResultBeforeFetch(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() @@ -649,6 +894,24 @@ func TestRunTasksKeepsGenericTasksOutsideDurableNetGuardProtocol(t *testing.T) { } } +func TestRunTasksRejectsUnknownDurableProtocolBeforeExecution(t *testing.T) { + oldClient := httpClient + defer func() { httpClient = oldClient }() + task := model.Task{ID: "task-unknown", LeaseID: "lease-unknown", Interpreter: "sh", Script: "must not run"} + data, _ := json.Marshal([]leasedAgentTask{{Task: task, DurableResult: true, DurableProtocol: "unknown-v1"}}) + httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + return testResponse(http.StatusOK, string(data)), nil + })} + runner := &countingTaskRunner{} + err := runTasks(agentConfig{Server: "http://lattice.test", NodeID: "node-a", Token: "secret"}, runner, beginFailingOutbox{}) + if err == nil || !strings.Contains(err.Error(), "unsupported protocol") { + t.Fatalf("error=%v", err) + } + if runner.calls != 0 { + t.Fatalf("unknown protocol executed: %d", runner.calls) + } +} + type completePublishingOutbox struct { task model.Task result *model.TaskResult diff --git a/go.mod b/go.mod index 25485ed..b9fa357 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require github.com/creack/pty v1.1.24 require ( github.com/gorilla/websocket v1.5.3 + golang.org/x/sys v0.43.0 google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 ) require ( golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect ) diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index c25cf74..bb41171 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -24,7 +24,7 @@ import ( const ( journalVersion = 1 maxDocumentSize = 4 << 20 - maxDiagnostic = 4096 + maxArtifactSize = maxDocumentSize ) var linechainBasenameRE = regexp.MustCompile(`^lattice-linechain-[0-9a-f]{20}\.json$`) @@ -34,12 +34,11 @@ var linechainBasenameRE = regexp.MustCompile(`^lattice-linechain-[0-9a-f]{20}\.j type Document struct { Version int `json:"version"` Operation string `json:"operation"` - ConfigDir string `json:"config_dir"` + ConfigDir string `json:"-"` FragmentBasename string `json:"fragment_basename,omitempty"` - FragmentPath string `json:"fragment_path"` - SidecarPath string `json:"sidecar_path"` + FragmentPath string `json:"-"` + SidecarPath string `json:"-"` PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` - PreviousSidecarSHA256 string `json:"previous_sidecar_sha256,omitempty"` Fragment *string `json:"fragment,omitempty"` Sidecar *string `json:"sidecar,omitempty"` FragmentSHA256 string `json:"fragment_sha256,omitempty"` @@ -88,6 +87,8 @@ type journal struct { SidecarOld string `json:"sidecar_old_sha256,omitempty"` FragmentNew string `json:"fragment_desired_sha256,omitempty"` SidecarNew string `json:"sidecar_desired_sha256,omitempty"` + CombinedNew string `json:"combined_desired_sha256"` + TaskScriptSHA string `json:"task_script_sha256"` FragmentHadOld bool `json:"fragment_had_old"` SidecarHadOld bool `json:"sidecar_had_old"` Phase string `json:"phase"` @@ -101,6 +102,8 @@ type Manager struct { configDir string sidecarPath string remove func(string) error + publishFile func(string, *string) error + writeJournal func(string, any) error syncDirectory func(string) error singBoxBinary string restartCommand []string @@ -113,12 +116,6 @@ func (m *Manager) ConfigureLayout(configDir, sidecarPath string) error { if !filepath.IsAbs(configDir) || !filepath.IsAbs(sidecarPath) { return fmt.Errorf("linechain layout paths must be absolute") } - for label, root := range map[string]string{"config": configDir, "sidecar": filepath.Dir(sidecarPath)} { - info, err := os.Lstat(root) - if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) || info.Mode().Perm()&0o022 != 0 { - return fmt.Errorf("linechain %s root is not a trusted private directory", label) - } - } resolvedConfig, err := filepath.EvalSymlinks(configDir) if err != nil { return fmt.Errorf("resolve linechain config directory: %w", err) @@ -127,6 +124,12 @@ func (m *Manager) ConfigureLayout(configDir, sidecarPath string) error { if err != nil { return fmt.Errorf("resolve linechain sidecar directory: %w", err) } + for label, root := range map[string]string{"config": resolvedConfig, "sidecar": resolvedSidecarParent} { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) || info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("linechain %s root is not a trusted private directory", label) + } + } configDir = filepath.Clean(resolvedConfig) sidecarPath = filepath.Join(filepath.Clean(resolvedSidecarParent), filepath.Base(sidecarPath)) if err := validateParents(filepath.Join(configDir, "placeholder")); err != nil { @@ -165,6 +168,18 @@ func (m *Manager) ConfigureCleanupForTest(remove func(string) error, syncDirecto m.syncDirectory = syncDirectory } +// ConfigureMutationForTest injects atomic publication and journal-write faults. +func (m *Manager) ConfigureMutationForTest(publishFile func(string, *string) error, writeJournal func(string, any) error) { + if publishFile == nil { + publishFile = publish + } + if writeJournal == nil { + writeJournal = writeJSON + } + m.publishFile = publishFile + m.writeJournal = writeJournal +} + // Open takes exclusive ownership of a private absolute transaction directory. func Open(dir string) (*Manager, error) { m, err := open(dir) @@ -205,6 +220,8 @@ func open(dir string) (*Manager, error) { } m := &Manager{dir: dir} m.remove = os.Remove + m.publishFile = publish + m.writeJournal = writeJSON m.syncDirectory = syncDir m.singBoxBinary = "sing-box" m.restartCommand = []string{"systemctl", "restart", "sing-box"} @@ -270,11 +287,22 @@ func (m *Manager) journalPath(taskID, leaseID string) string { // Apply reads and applies one document. It is intended for the early-exit // helper mode and obtains task identity from the runner's minimal environment. -func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string) error { +func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskScriptSHA string) error { if strings.TrimSpace(taskID) == "" || strings.TrimSpace(leaseID) == "" { return fmt.Errorf("task and lease identity are required") } - dec := json.NewDecoder(io.LimitReader(r, maxDocumentSize+1)) + scriptBinding := strings.TrimSpace(taskScriptSHA) + if !validSHA(scriptBinding) { + return fmt.Errorf("linechain task script binding is required and must be lowercase SHA-256") + } + raw, err := io.ReadAll(io.LimitReader(r, maxDocumentSize+1)) + if err != nil { + return fmt.Errorf("read linechain document: %w", err) + } + if len(raw) > maxDocumentSize { + return fmt.Errorf("linechain document exceeds %d bytes", maxDocumentSize) + } + dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() var wire wireDocumentV2 if err := dec.Decode(&wire); err != nil { @@ -301,15 +329,11 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := m.validateDocument(d); err != nil { return err } - mergedSidecar, err := mergeSidecar(sidecarCurrent(m.sidecarPath), d.Sidecar) - if err != nil { - return err - } - d.Sidecar = mergedSidecar - d = BindDocument(d) path := m.journalPath(taskID, leaseID) if _, err := os.Lstat(path); err == nil { - return m.recoverOne(ctx, path, nil, "") + return m.recoverOne(ctx, path, nil, "", scriptBinding) + } else if !errors.Is(err, os.ErrNotExist) { + return err } fragmentOld, fragmentHad, err := readCurrent(d.FragmentPath) @@ -323,67 +347,64 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID string if err := requirePrevious("fragment", fragmentOld, fragmentHad, d.PreviousFragmentSHA256); err != nil { return err } + mergedSidecar, err := mergeSidecar(sidecarOld, sidecarHad, d.Sidecar) + if err != nil { + return err + } + d.Sidecar = mergedSidecar + d = BindDocument(d) j := journal{Version: journalVersion, TaskID: taskID, LeaseID: leaseID, FragmentPath: d.FragmentPath, SidecarPath: d.SidecarPath, FragmentOld: digestMaybe(fragmentOld, fragmentHad), SidecarOld: digestMaybe(sidecarOld, sidecarHad), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad, - FragmentNew: digestPtr(d.Fragment), SidecarNew: digestPtr(d.Sidecar), Phase: "prepared"} + FragmentNew: digestPtr(d.Fragment), SidecarNew: digestPtr(d.Sidecar), CombinedNew: d.CombinedSHA256, TaskScriptSHA: scriptBinding, Phase: "prepared"} if err := m.writeBackup(path+".fragment.old", fragmentOld, fragmentHad); err != nil { return err } if err := m.writeBackup(path+".sidecar.old", sidecarOld, sidecarHad); err != nil { return err } - if err := writeJSON(path, j); err != nil { + if err := m.writeJournal(path, j); err != nil { return err } - if err := publish(d.FragmentPath, d.Fragment); err != nil { + if err := m.publishFile(d.FragmentPath, d.Fragment); err != nil { return m.rollback(ctx, path, &j, d, fmt.Errorf("publish fragment: %w", err)) } j.Phase = "fragment_published" - if err := writeJSON(path, j); err != nil { + if err := m.writeJournal(path, j); err != nil { return err } - if err := publish(d.SidecarPath, d.Sidecar); err != nil { + if err := m.publishFile(d.SidecarPath, d.Sidecar); err != nil { return m.rollback(ctx, path, &j, d, fmt.Errorf("publish sidecar: %w", err)) } j.Phase = "pair_published" - if err := writeJSON(path, j); err != nil { + if err := m.writeJournal(path, j); err != nil { return err } if err := m.checkRestartVerify(ctx, d); err != nil { return m.rollback(ctx, path, &j, d, err) } j.Phase = "desired_verified" - return writeJSON(path, j) + return m.writeJournal(path, j) } -func sidecarCurrent(path string) *string { - b, err := os.ReadFile(path) - if err != nil { - return nil - } - s := string(b) - return &s -} - -func mergeSidecar(current, desired *string) (*string, error) { +func mergeSidecar(current []byte, currentExists bool, desired *string) (*string, error) { if desired == nil { return nil, nil } merged := map[string]any{} - if current != nil { - if err := json.Unmarshal([]byte(*current), &merged); err != nil { + if currentExists { + if err := json.Unmarshal(current, &merged); err != nil { return nil, fmt.Errorf("decode current semantic sidecar: %w", err) } + if err := validateSidecarObject("current", merged); err != nil { + return nil, err + } } want := map[string]any{} if err := json.Unmarshal([]byte(*desired), &want); err != nil { return nil, fmt.Errorf("decode desired semantic sidecar: %w", err) } - if want["schema"] != "lattice.singbox-metadata.v2" { - return nil, fmt.Errorf("desired semantic sidecar has unsupported schema") - } - if _, ok := want["inbounds"].([]any); !ok { - return nil, fmt.Errorf("desired semantic sidecar inbounds must be an array") + if err := validateSidecarObject("desired", want); err != nil { + return nil, err } for key, value := range want { merged[key] = value @@ -397,8 +418,33 @@ func mergeSidecar(current, desired *string) (*string, error) { return &s, nil } +func validateSidecarObject(label string, value map[string]any) error { + if value == nil { + return fmt.Errorf("%s semantic sidecar must be a non-null object", label) + } + if value["schema"] != "lattice.singbox-metadata.v2" { + return fmt.Errorf("%s semantic sidecar has unsupported schema", label) + } + if _, ok := value["inbounds"].([]any); !ok { + return fmt.Errorf("%s semantic sidecar inbounds must be an array", label) + } + return nil +} + // JournalRef is the bounded identity needed to cross-check outbox authority. -type JournalRef struct{ TaskID, LeaseID string } +type JournalRef struct { + TaskID, LeaseID string + Phase string + Result *model.TaskResult + Terminal bool + TaskScriptSHA string + CombinedSHA256 string + JournalSHA256 string +} + +type RecoveryAuthority struct { + TaskScriptSHA, CombinedSHA256, Phase, ResultSHA256, JournalSHA256 string +} func (m *Manager) Snapshot() ([]JournalRef, error) { entries, err := os.ReadDir(m.dir) @@ -406,6 +452,7 @@ func (m *Manager) Snapshot() ([]JournalRef, error) { return nil, err } refs := make([]JournalRef, 0, len(entries)) + seen := make(map[string]struct{}, len(entries)) for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { continue @@ -417,11 +464,100 @@ func (m *Manager) Snapshot() ([]JournalRef, error) { if err != nil { return nil, err } - refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID}) + key := j.TaskID + "\x00" + j.LeaseID + if _, ok := seen[key]; ok { + return nil, fmt.Errorf("duplicate linechain journal identity: %s", j.TaskID) + } + seen[key] = struct{}{} + if entry.Name() != filepath.Base(m.journalPath(j.TaskID, j.LeaseID)) { + return nil, fmt.Errorf("linechain journal filename does not match task and lease: %s", entry.Name()) + } + if err := m.validateJournal(filepath.Join(m.dir, entry.Name()), j); err != nil { + return nil, err + } + refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID, Phase: j.Phase, Result: j.Result, Terminal: journalTerminal(j.Phase), TaskScriptSHA: j.TaskScriptSHA, CombinedSHA256: j.CombinedNew, JournalSHA256: journalSHA(j)}) } return refs, nil } +func (m *Manager) validateJournal(path string, j journal) error { + if !m.Configured() { + return fmt.Errorf("linechain journal exists before runtime layout is configured") + } + if filepath.Clean(path) != m.journalPath(j.TaskID, j.LeaseID) { + return fmt.Errorf("linechain journal filename does not match task and lease") + } + fragmentName := filepath.Base(j.FragmentPath) + if filepath.Clean(j.FragmentPath) != filepath.Join(m.configDir, fragmentName) || !linechainBasenameRE.MatchString(fragmentName) { + return fmt.Errorf("linechain journal fragment path is outside local authority") + } + if filepath.Clean(j.SidecarPath) != m.sidecarPath { + return fmt.Errorf("linechain journal sidecar path is outside local authority") + } + validPhase := map[string]bool{ + "prepared": true, "fragment_published": true, "pair_published": true, "desired_verified": true, + "old_restored": true, "terminal_desired": true, "terminal_old": true, + } + if !validPhase[j.Phase] { + return fmt.Errorf("linechain journal has unknown phase %q", j.Phase) + } + terminal := journalTerminal(j.Phase) + if terminal != (j.Result != nil) { + return fmt.Errorf("linechain journal phase/result shape is inconsistent") + } + if j.Result != nil && (j.Result.TaskID != j.TaskID || j.Result.LeaseID != j.LeaseID || j.Result.FinishedAt.IsZero()) { + return fmt.Errorf("linechain journal terminal result does not match its identity") + } + if (j.FragmentHadOld && !validSHA(j.FragmentOld)) || (!j.FragmentHadOld && j.FragmentOld != "") || + (j.SidecarHadOld && !validSHA(j.SidecarOld)) || (!j.SidecarHadOld && j.SidecarOld != "") || + (j.FragmentNew != "" && !validSHA(j.FragmentNew)) || !validSHA(j.SidecarNew) || !validSHA(j.CombinedNew) || !validSHA(j.TaskScriptSHA) { + return fmt.Errorf("linechain journal artifact digest shape is invalid") + } + if terminal { + return nil + } + for _, backup := range []struct { + path string + exists bool + want string + }{ + {path: path + ".fragment.old", exists: j.FragmentHadOld, want: j.FragmentOld}, + {path: path + ".sidecar.old", exists: j.SidecarHadOld, want: j.SidecarOld}, + } { + data, exists, err := readValidatedFile(backup.path, true) + if err != nil { + return err + } + if exists != backup.exists || (exists && digest(data) != backup.want) { + return fmt.Errorf("linechain journal backup shape does not match authority") + } + } + return nil +} + +func journalTerminal(phase string) bool { + return phase == "terminal_desired" || phase == "terminal_old" +} + +func resultSHA(result *model.TaskResult) string { + if result == nil { + return "" + } + b, err := json.Marshal(result) + if err != nil { + return "" + } + return digest(b) +} + +func journalSHA(j journal) string { + b, err := json.Marshal(j) + if err != nil { + return "" + } + return digest(b) +} + func (m *Manager) validateDocument(d Document) error { if d.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", d.Version) @@ -471,7 +607,7 @@ func (m *Manager) validateDocument(d Document) error { return fmt.Errorf("fragment and sidecar paths must differ") } fragmentName := filepath.Base(d.FragmentPath) - if filepath.Dir(filepath.Clean(d.FragmentPath)) != configDir || !strings.HasPrefix(fragmentName, "lattice-linechain-") || filepath.Ext(fragmentName) != ".json" { + if filepath.Dir(filepath.Clean(d.FragmentPath)) != configDir || !linechainBasenameRE.MatchString(fragmentName) { return fmt.Errorf("fragment path is outside the server-owned linechain config namespace") } wantSidecar := m.sidecarPath @@ -493,16 +629,16 @@ func (m *Manager) validateDocument(d Document) error { } func (m *Manager) checkRestartVerify(ctx context.Context, d Document) error { - if out, err := m.run(ctx, m.singBoxBinary, "check", "-C", m.configDir); err != nil { - return fmt.Errorf("sing-box check failed: %s", bounded(out)) + if _, err := m.run(ctx, m.singBoxBinary, "check", "-C", m.configDir); err != nil { + return fmt.Errorf("sing-box check failed") } restart := m.restartCommand - if out, err := m.run(ctx, restart[0], restart[1:]...); err != nil { - return fmt.Errorf("sing-box restart failed: %s", bounded(out)) + if _, err := m.run(ctx, restart[0], restart[1:]...); err != nil { + return fmt.Errorf("sing-box restart failed") } verify := m.verifyCommand - if out, err := m.run(ctx, verify[0], verify[1:]...); err != nil { - return fmt.Errorf("sing-box active verification failed: %s", bounded(out)) + if _, err := m.run(ctx, verify[0], verify[1:]...); err != nil { + return fmt.Errorf("sing-box active verification failed") } return nil } @@ -530,11 +666,26 @@ func (m *Manager) ResolveAfterRun(ctx context.Context, task model.Task, result m if !pairMatches(j, true) { return result, fmt.Errorf("desired linechain pair is not exact") } - j.Phase = "terminal_desired" + if verifyErr := m.checkRestartVerify(ctx, Document{}); verifyErr == nil { + j.Phase = "terminal_desired" + } else { + if err := m.restoreOld(path, &j); err != nil { + return result, fmt.Errorf("post-run desired runtime verification failed: %v; restore old pair: %w", verifyErr, err) + } + if err := m.checkRestartVerify(ctx, Document{}); err != nil { + return result, fmt.Errorf("post-run desired runtime verification failed: %v; old runtime recovery failed: %w", verifyErr, err) + } + result.ExitCode = -1 + result.Error = "desired linechain runtime verification failed after helper return; exact old pair restored and verified" + j.Phase = "terminal_old" + } } else { if err := m.restoreOld(path, &j); err != nil { return result, err } + if err := m.checkRestartVerify(ctx, Document{}); err != nil { + return result, fmt.Errorf("verify restored old linechain runtime after helper failure: %w", err) + } if result.ExitCode == 0 { result.ExitCode = -1 } @@ -544,7 +695,7 @@ func (m *Manager) ResolveAfterRun(ctx context.Context, task model.Task, result m j.Phase = "terminal_old" } j.Result = &result - if err := writeJSON(path, j); err != nil { + if err := m.writeJournal(path, j); err != nil { return result, err } return result, nil @@ -554,6 +705,10 @@ func (m *Manager) ResolveAfterRun(ctx context.Context, task model.Task, result m // complete is called only with a stable exact result; cleanup follows its // confirmed durable completion. func (m *Manager) RequireRecovered(ctx context.Context, complete func(model.TaskResult) error, nodeID string) error { + return m.RequireRecoveredAuthorized(ctx, complete, nodeID, nil) +} + +func (m *Manager) RequireRecoveredAuthorized(ctx context.Context, complete func(model.TaskResult) error, nodeID string, authority map[string]RecoveryAuthority) error { entries, err := os.ReadDir(m.dir) if err != nil { return err @@ -562,24 +717,60 @@ func (m *Manager) RequireRecovered(ctx context.Context, complete func(model.Task if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { continue } - if err := m.recoverOne(ctx, filepath.Join(m.dir, entry.Name()), complete, nodeID); err != nil { + path := filepath.Join(m.dir, entry.Name()) + var expected *RecoveryAuthority + if authority != nil { + j, err := readJournal(path) + if err != nil { + return err + } + value, ok := authority[j.TaskID+"\x00"+j.LeaseID] + if !ok { + return fmt.Errorf("linechain recovery journal is outside captured authority") + } + expected = &value + } + if err := m.recoverOne(ctx, path, complete, nodeID, "", expected); err != nil { return err } } return nil } -func (m *Manager) recoverOne(ctx context.Context, path string, complete func(model.TaskResult) error, nodeID string) error { +func (m *Manager) recoverOne(ctx context.Context, path string, complete func(model.TaskResult) error, nodeID, expectedScriptSHA string, expected ...*RecoveryAuthority) error { j, err := readJournal(path) if err != nil { return err } + if err := m.validateJournal(path, j); err != nil { + return err + } + if expectedScriptSHA != "" && j.TaskScriptSHA != expectedScriptSHA { + return fmt.Errorf("linechain retry task script does not match journal authority") + } + if len(expected) > 0 && expected[0] != nil { + want := expected[0] + if journalSHA(j) != want.JournalSHA256 || j.TaskScriptSHA != want.TaskScriptSHA || j.CombinedNew != want.CombinedSHA256 || j.Phase != want.Phase || resultSHA(j.Result) != want.ResultSHA256 { + return fmt.Errorf("linechain journal changed after authority capture") + } + } if j.Result == nil { result := model.TaskResult{TaskID: j.TaskID, LeaseID: j.LeaseID, NodeID: nodeID, ExitCode: -1, Error: "linechain transaction interrupted; exact old pair restored", FinishedAt: time.Now().UTC()} if j.Phase == "desired_verified" && pairMatches(j, true) { - result.ExitCode = 0 - result.Error = "" - j.Phase = "terminal_desired" + if verifyErr := m.checkRestartVerify(ctx, Document{}); verifyErr == nil { + result.ExitCode = 0 + result.Error = "" + j.Phase = "terminal_desired" + } else { + if err := m.restoreOld(path, &j); err != nil { + return fmt.Errorf("recover desired runtime verification failed: %v; restore old pair: %w", verifyErr, err) + } + if err := m.checkRestartVerify(ctx, Document{}); err != nil { + return fmt.Errorf("recover desired runtime verification failed: %v; old runtime recovery failed: %w", verifyErr, err) + } + result.Error = "recovered desired runtime verification failed; exact old pair restored and verified" + j.Phase = "terminal_old" + } } else { if err := m.restoreOld(path, &j); err != nil { return err @@ -590,7 +781,7 @@ func (m *Manager) recoverOne(ctx context.Context, path string, complete func(mod j.Phase = "terminal_old" } j.Result = &result - if err := writeJSON(path, j); err != nil { + if err := m.writeJournal(path, j); err != nil { return err } } @@ -618,7 +809,7 @@ func (m *Manager) rollback(ctx context.Context, path string, j *journal, d Docum return fmt.Errorf("%v; rollback failed: %w", cause, err) } if err := m.checkRestartVerify(ctx, d); err != nil { - return fmt.Errorf("%v; exact old pair restored but service recovery failed: %s", cause, bounded([]byte(err.Error()))) + return fmt.Errorf("%v; exact old pair restored but service recovery failed: %v", cause, err) } return cause } @@ -630,18 +821,21 @@ func (m *Manager) restoreOld(path string, j *journal) error { want string }{{j.FragmentPath, path + ".fragment.old", j.FragmentHadOld, j.FragmentOld}, {j.SidecarPath, path + ".sidecar.old", j.SidecarHadOld, j.SidecarOld}} { if a.had { - data, err := os.ReadFile(a.backup) + data, exists, err := readValidatedFile(a.backup, true) if err != nil { return err } + if !exists { + return fmt.Errorf("required linechain backup is missing") + } if digest(data) != a.want { return fmt.Errorf("backup digest mismatch") } s := string(data) - if err := publish(a.dst, &s); err != nil { + if err := m.publishFile(a.dst, &s); err != nil { return err } - } else if err := publish(a.dst, nil); err != nil { + } else if err := m.publishFile(a.dst, nil); err != nil { return err } } @@ -649,7 +843,7 @@ func (m *Manager) restoreOld(path string, j *journal) error { return fmt.Errorf("restored pair digest mismatch") } j.Phase = "old_restored" - return writeJSON(path, *j) + return m.writeJournal(path, *j) } func pairMatches(j journal, desired bool) bool { @@ -665,17 +859,29 @@ func pairMatches(j journal, desired bool) bool { } func readCurrent(path string) ([]byte, bool, error) { - info, err := os.Lstat(path) + return readValidatedFile(path, false) +} + +func readValidatedFile(path string, private bool) ([]byte, bool, error) { + f, err := openNoFollow(path) if errors.Is(err, os.ErrNotExist) { return nil, false, nil } if err != nil { return nil, false, err } - if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || !ownedPath(info) { - return nil, false, fmt.Errorf("artifact is not a regular file: %s", path) + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, false, err + } + if !info.Mode().IsRegular() || !ownedPath(info) || (private && info.Mode().Perm()&0o077 != 0) { + return nil, false, fmt.Errorf("linechain file is not a trusted regular file: %s", path) + } + b, err := io.ReadAll(io.LimitReader(f, maxArtifactSize+1)) + if err == nil && len(b) > maxArtifactSize { + return nil, false, fmt.Errorf("linechain file exceeds %d bytes: %s", maxArtifactSize, path) } - b, err := os.ReadFile(path) return b, true, err } func requirePrevious(name string, data []byte, exists bool, want string) error { @@ -707,13 +913,6 @@ func validSHA(s string) bool { b, e := hex.DecodeString(s) return e == nil && len(b) == sha256.Size && s == strings.ToLower(s) } -func bounded(b []byte) string { - b = bytes.TrimSpace(b) - if len(b) > maxDiagnostic { - b = b[:maxDiagnostic] - } - return string(b) -} func (m *Manager) writeBackup(path string, data []byte, exists bool) error { if !exists { return nil @@ -794,19 +993,36 @@ func writeJSON(path string, v any) error { return writeFile(path, b) } func readJournal(path string) (journal, error) { - info, err := os.Lstat(path) + f, err := openNoFollow(path) + if err != nil { + return journal{}, err + } + defer f.Close() + info, err := f.Stat() if err != nil { return journal{}, err } - if !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + if !info.Mode().IsRegular() || !ownedPath(info) || info.Mode().Perm()&0o077 != 0 { return journal{}, fmt.Errorf("invalid linechain journal") } - b, err := os.ReadFile(path) + b, err := io.ReadAll(io.LimitReader(f, maxDocumentSize+1)) if err != nil { return journal{}, err } + if len(b) > maxDocumentSize { + return journal{}, fmt.Errorf("linechain journal exceeds %d bytes", maxDocumentSize) + } var j journal - if err = json.Unmarshal(b, &j); err != nil { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err = dec.Decode(&j); err != nil { + return j, err + } + var trailing any + if err = dec.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return j, fmt.Errorf("linechain journal has trailing data") + } return j, err } if j.Version != journalVersion || j.TaskID == "" || j.LeaseID == "" { diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index ab9b8a6..863a451 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -1,10 +1,12 @@ package linechain import ( + "bytes" "context" "encoding/json" "errors" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -26,7 +28,7 @@ func TestApplyCreateResolveAndCleanup(t *testing.T) { t.Fatal(err) } fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}` - sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"credential":"sidecar-secret"}` doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, Sidecar: &sidecar} applyDoc(t, m, doc, "task-a", "lease-a") result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()}) @@ -37,7 +39,7 @@ func TestApplyCreateResolveAndCleanup(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(journalBytes), fragment) { + if strings.Contains(string(journalBytes), fragment) || strings.Contains(string(journalBytes), "sidecar-secret") { t.Fatal("journal contains credential-bearing desired bytes") } if err := m.Cleanup("task-a", "lease-a"); err != nil { @@ -72,7 +74,7 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { if err := os.Symlink(filepath.Join(dir, "target"), fragmentPath); err != nil { t.Fatal(err) } - if err := applyDocErr(m, doc, "task-b", "lease-b"); err == nil || !strings.Contains(err.Error(), "not a regular file") { + if err := applyDocErr(m, doc, "task-b", "lease-b"); err == nil { t.Fatalf("unexpected error: %v", err) } } @@ -89,7 +91,7 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { if err := m.ConfigureCommands("false", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - oldFragment, oldSidecar := "old-fragment", `{"ordinary":"old"}` + oldFragment, oldSidecar := "old-fragment", `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"old"}` for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} { if err := os.WriteFile(path, []byte(data), 0o600); err != nil { t.Fatal(err) @@ -107,15 +109,19 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { m, dir := testManager(t) mustMkdir(t, filepath.Join(dir, "conf")) - fragmentPath := filepath.Join(dir, "conf", "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(dir, "lattice-metadata.json") if err := m.ConfigureLayout(filepath.Join(dir, "conf"), sidecarPath); err != nil { t.Fatal(err) } + fragmentPath := filepath.Join(m.configDir, "lattice-linechain-0123456789abcdef0123.json") + sidecarPath = m.sidecarPath if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - j := journal{Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, Phase: "prepared"} + j := journal{ + Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, + FragmentNew: digest([]byte("new-fragment")), SidecarNew: digest([]byte("new-sidecar")), CombinedNew: digest([]byte("combined")), TaskScriptSHA: digest(nil), Phase: "prepared", + } path := m.journalPath(j.TaskID, j.LeaseID) if err := writeJSON(path, j); err != nil { t.Fatal(err) @@ -189,6 +195,12 @@ func TestValidateDocumentBindsOwnedPaths(t *testing.T) { if err := m.validateDocument(invalid); err == nil { t.Fatal("unowned sidecar accepted") } + invalid = valid + invalid.FragmentBasename = "lattice-linechain-not-hex.json" + invalid.FragmentPath = filepath.Join(configDir, invalid.FragmentBasename) + if err := m.validateDocument(invalid); err == nil { + t.Fatal("malformed fragment basename accepted by internal validation") + } } func TestResolvePreflightFailureWithoutJournal(t *testing.T) { @@ -215,16 +227,37 @@ func TestApplyRejectsTrailingAndLegacyWireFields(t *testing.T) { d := BindDocument(Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) base, _ := json.Marshal(wireDocumentV2{Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, Fragment: d.Fragment, Sidecar: d.Sidecar, FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256}) for name, raw := range map[string]string{ - "trailing": string(base) + ` {}`, - "legacy path": strings.TrimSuffix(string(base), "}") + `,"config_dir":"/tmp"}`, - "legacy sidecar digest": strings.TrimSuffix(string(base), "}") + `,"previous_sidecar_sha256":"` + strings.Repeat("0", 64) + `"}`, + "trailing": string(base) + ` {}`, + "legacy config path": strings.TrimSuffix(string(base), "}") + `,"config_dir":""}`, + "legacy fragment path": strings.TrimSuffix(string(base), "}") + `,"fragment_path":""}`, + "legacy sidecar path": strings.TrimSuffix(string(base), "}") + `,"sidecar_path":""}`, + "legacy previous sidecar digest": strings.TrimSuffix(string(base), "}") + `,"previous_sidecar_sha256":""}`, } { t.Run(name, func(t *testing.T) { - if err := m.Apply(context.Background(), strings.NewReader(raw), "task-"+name, "lease"); err == nil { + if err := m.Apply(context.Background(), strings.NewReader(raw), "task-"+name, "lease", digest([]byte("script"))); err == nil { t.Fatal("invalid wire accepted") } }) } + t.Run("oversized whitespace", func(t *testing.T) { + raw := string(base) + strings.Repeat(" ", maxDocumentSize-len(base)+1) + if err := m.Apply(context.Background(), strings.NewReader(raw), "task-oversized", "lease", digest([]byte("script"))); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("oversized wire error = %v", err) + } + }) +} + +func TestApplyRequiresStrictTaskScriptBinding(t *testing.T) { + m, _ := testManager(t) + for name, binding := range map[string]string{ + "missing": "", "short": "abcd", "uppercase": strings.Repeat("A", 64), + } { + t.Run(name, func(t *testing.T) { + if err := m.Apply(context.Background(), strings.NewReader(`{}`), "task", "lease", binding); err == nil || !strings.Contains(err.Error(), "script binding") { + t.Fatalf("binding error=%v", err) + } + }) + } } func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { @@ -232,7 +265,7 @@ func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { configDir := filepath.Join(dir, "conf") mustMkdir(t, configDir) sidecarPath := filepath.Join(dir, "lattice-metadata.json") - if err := os.WriteFile(sidecarPath, []byte(`{"ordinary":{"owner":"sb"},"inbounds":[{"tag":"old"}]}`), 0o600); err != nil { + if err := os.WriteFile(sidecarPath, []byte(`{"schema":"lattice.singbox-metadata.v2","ordinary":{"owner":"sb"},"inbounds":[{"tag":"old"}]}`), 0o600); err != nil { t.Fatal(err) } if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { @@ -257,7 +290,7 @@ func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { } } -func TestConfigureLayoutRejectsSymlinkRoot(t *testing.T) { +func TestConfigureLayoutAcceptsTrustedAliasAndRejectsHostileTarget(t *testing.T) { m, dir := testManager(t) realDir := filepath.Join(dir, "real") mustMkdir(t, realDir) @@ -265,18 +298,660 @@ func TestConfigureLayoutRejectsSymlinkRoot(t *testing.T) { if err := os.Symlink(realDir, link); err != nil { t.Fatal(err) } - if err := m.ConfigureLayout(link, filepath.Join(dir, "meta.json")); err == nil { - t.Fatal("symlink config root accepted") + metaReal := filepath.Join(dir, "meta-real") + mustMkdir(t, metaReal) + metaAlias := filepath.Join(dir, "meta-alias") + if err := os.Symlink(metaReal, metaAlias); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(link, filepath.Join(metaAlias, "meta.json")); err != nil { + t.Fatalf("trusted config alias rejected: %v", err) + } + wantConfig, _ := filepath.EvalSymlinks(realDir) + wantMeta, _ := filepath.EvalSymlinks(metaReal) + if m.configDir != wantConfig || m.sidecarPath != filepath.Join(wantMeta, "meta.json") { + t.Fatalf("trusted aliases not canonicalized: config=%s sidecar=%s", m.configDir, m.sidecarPath) + } + badTarget := filepath.Join(dir, "bad-target") + mustMkdir(t, badTarget) + if err := os.Chmod(badTarget, 0o777); err != nil { + t.Fatal(err) + } + badAlias := filepath.Join(dir, "bad-alias") + if err := os.Symlink(badTarget, badAlias); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(badAlias, filepath.Join(dir, "meta.json")); err == nil { + t.Fatal("alias to writable config root accepted") + } +} + +func TestApplyRejectsSymlinkAndInvalidCurrentSidecar(t *testing.T) { + for name, setup := range map[string]func(string) error{ + "symlink": func(path string) error { + target := path + ".target" + if err := os.WriteFile(target, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[]}`), 0o600); err != nil { + return err + } + return os.Symlink(target, path) + }, + "null": func(path string) error { return os.WriteFile(path, []byte(`null`), 0o600) }, + "wrong schema": func(path string) error { return os.WriteFile(path, []byte(`{"schema":"v1","inbounds":[]}`), 0o600) }, + "wrong inbounds": func(path string) error { + return os.WriteFile(path, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":{}}`), 0o600) + }, + } { + t.Run(name, func(t *testing.T) { + m, dir := testManager(t) + configDir := filepath.Join(dir, "conf") + mustMkdir(t, configDir) + sidecarPath := filepath.Join(dir, "lattice-metadata.json") + if err := setup(sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar} + if err := applyDocErr(m, d, "task-"+name, "lease"); err == nil { + t.Fatal("invalid current sidecar accepted") + } + }) + } +} + +func TestSnapshotRejectsRenamedAndDuplicateJournalIdentity(t *testing.T) { + for _, duplicate := range []bool{false, true} { + t.Run(map[bool]string{false: "renamed", true: "duplicate"}[duplicate], func(t *testing.T) { + m, _ := testManager(t) + j := journal{Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", Phase: "prepared"} + canonical := m.journalPath(j.TaskID, j.LeaseID) + if duplicate { + if err := writeJSON(canonical, j); err != nil { + t.Fatal(err) + } + } + if err := writeJSON(filepath.Join(m.dir, "zz-renamed.json"), j); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("corrupt journal namespace accepted") + } + }) + } +} + +func TestSnapshotRejectsSymlinkFIFOOwnerAndMode(t *testing.T) { + t.Run("symlink", func(t *testing.T) { + m, _ := testManager(t) + target := filepath.Join(t.TempDir(), "target.json") + if err := os.WriteFile(target, []byte(`{"version":1,"task_id":"secret","lease_id":"secret"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(m.dir, "symlink.json")); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("journal symlink accepted") + } + }) + t.Run("fifo", func(t *testing.T) { + m, _ := testManager(t) + fifo := filepath.Join(m.dir, "fifo.json") + if err := exec.Command("mkfifo", fifo).Run(); err != nil { + t.Skipf("mkfifo unavailable: %v", err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("journal FIFO accepted") + } + }) + t.Run("mode", func(t *testing.T) { + m, _ := testManager(t) + j := journal{Version: journalVersion, TaskID: "task-mode", LeaseID: "lease"} + path := m.journalPath(j.TaskID, j.LeaseID) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("non-private journal mode accepted") + } + }) + if os.Geteuid() == 0 { + t.Run("owner", func(t *testing.T) { + m, _ := testManager(t) + j := journal{Version: journalVersion, TaskID: "task-owner", LeaseID: "lease"} + path := m.journalPath(j.TaskID, j.LeaseID) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + if err := os.Chown(path, 65534, 65534); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("wrong-owner journal accepted") + } + }) + } +} + +func TestBindDocumentMatchesCrossContractDigestVector(t *testing.T) { + fragment := "{\"outbounds\":[]}\n" + sidecar := "{\"inbounds\":[],\"schema\":\"lattice.singbox-metadata.v2\"}\n" + d := BindDocument(Document{Fragment: &fragment, Sidecar: &sidecar}) + if d.FragmentSHA256 != "0f4c04e3dff1fb9208c856b6de82f766bf6d99ff25ef30c68e076075a8d44b28" || + d.SidecarSHA256 != "f2eee32252fab6c99a728ebb1f16122798aff8609ace953f5f338d8f80b6b979" || + d.CombinedSHA256 != "155ec43b76241d49eac9d3a1415df33d84571f02363ac42bae04b0627e2ab848" { + t.Fatalf("cross-contract digest drift: %+v", d) + } +} + +func TestRecoveryPhasesRestoreOrCommitExactPair(t *testing.T) { + for _, tc := range []struct { + phase string + wantSuccess bool + }{ + {phase: "prepared"}, + {phase: "fragment_published"}, + {phase: "pair_published"}, + {phase: "desired_verified", wantSuccess: true}, + } { + t.Run(tc.phase, func(t *testing.T) { + m, _ := testManager(t) + artifactRoot := t.TempDir() + configDir := filepath.Join(artifactRoot, "conf") + mustMkdir(t, configDir) + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") + sidecarPath := filepath.Join(artifactRoot, "lattice-metadata.json") + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + fragmentPath = filepath.Join(m.configDir, filepath.Base(fragmentPath)) + sidecarPath = m.sidecarPath + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + oldFragment := []byte("old-fragment") + oldSidecar := []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}`) + newFragment := []byte("new-fragment") + newSidecar := []byte(`{"inbounds":[],"schema":"lattice.singbox-metadata.v2"}` + "\n") + currentFragment, currentSidecar := newFragment, newSidecar + if tc.phase == "prepared" { + currentFragment, currentSidecar = oldFragment, oldSidecar + } else if tc.phase == "fragment_published" { + currentSidecar = oldSidecar + } + if err := os.WriteFile(fragmentPath, currentFragment, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sidecarPath, currentSidecar, 0o600); err != nil { + t.Fatal(err) + } + j := journal{ + Version: journalVersion, TaskID: "task-" + tc.phase, LeaseID: "lease", FragmentPath: fragmentPath, SidecarPath: sidecarPath, + FragmentOld: digest(oldFragment), SidecarOld: digest(oldSidecar), FragmentNew: digest(newFragment), SidecarNew: digest(newSidecar), + CombinedNew: digest(append(append(append([]byte{}, newFragment...), 0), newSidecar...)), TaskScriptSHA: digest(nil), + FragmentHadOld: true, SidecarHadOld: true, Phase: tc.phase, + } + path := m.journalPath(j.TaskID, j.LeaseID) + if err := m.writeBackup(path+".fragment.old", oldFragment, true); err != nil { + t.Fatal(err) + } + if err := m.writeBackup(path+".sidecar.old", oldSidecar, true); err != nil { + t.Fatal(err) + } + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + var got model.TaskResult + if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { got = result; return nil }, "node-a"); err != nil { + t.Fatal(err) + } + if (got.ExitCode == 0) != tc.wantSuccess { + t.Fatalf("result=%+v", got) + } + if tc.wantSuccess { + assertFile(t, fragmentPath, string(newFragment)) + assertFile(t, sidecarPath, string(newSidecar)) + } else { + assertFile(t, fragmentPath, string(oldFragment)) + assertFile(t, sidecarPath, string(oldSidecar)) + } + }) + } +} + +func TestJournalAuthorityRejectsCorruptSemanticsAndUnconfiguredRecovery(t *testing.T) { + t.Run("unconfigured empty allowed", func(t *testing.T) { + m, _ := testManager(t) + if refs, err := m.Snapshot(); err != nil || len(refs) != 0 { + t.Fatalf("empty unconfigured snapshot = %+v, %v", refs, err) + } + }) + t.Run("unconfigured journal blocked", func(t *testing.T) { + m, _ := testManager(t) + j := journal{Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", Phase: "prepared"} + if err := writeJSON(m.journalPath(j.TaskID, j.LeaseID), j); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil || !strings.Contains(err.Error(), "before runtime layout") { + t.Fatalf("error=%v", err) + } + }) + for name, mutate := range map[string]func(*journal){ + "phase": func(j *journal) { j.Phase = "invented" }, + "path": func(j *journal) { j.FragmentPath = filepath.Join(t.TempDir(), filepath.Base(j.FragmentPath)) }, + "digest": func(j *journal) { j.SidecarNew = "bad" }, + "result": func(j *journal) { + j.Phase = "terminal_desired" + j.Result = &model.TaskResult{TaskID: "wrong", LeaseID: j.LeaseID, FinishedAt: time.Now().UTC()} + }, + } { + t.Run(name, func(t *testing.T) { + m, path := appliedJournal(t, "task-"+name, "lease") + j, err := readJournal(path) + if err != nil { + t.Fatal(err) + } + mutate(&j) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("corrupt journal semantics accepted") + } + }) + } + t.Run("unknown field", func(t *testing.T) { + m, path := appliedJournal(t, "task-unknown-field", "lease") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + raw = append(bytes.TrimSuffix(raw, []byte("}")), []byte(`,"unexpected":true}`)...) + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err := m.Snapshot(); err == nil { + t.Fatal("unknown journal field accepted") + } + }) +} + +func TestAuthorizedRecoveryRejectsJournalSwapAfterSnapshot(t *testing.T) { + m, path := appliedJournal(t, "task-swap", "lease") + refs, err := m.Snapshot() + if err != nil || len(refs) != 1 { + t.Fatalf("snapshot=%+v err=%v", refs, err) + } + ref := refs[0] + authority := map[string]RecoveryAuthority{ + ref.TaskID + "\x00" + ref.LeaseID: {TaskScriptSHA: ref.TaskScriptSHA, CombinedSHA256: ref.CombinedSHA256, Phase: ref.Phase, ResultSHA256: resultSHA(ref.Result), JournalSHA256: ref.JournalSHA256}, + } + j, err := readJournal(path) + if err != nil { + t.Fatal(err) + } + j.CombinedNew = digest([]byte("swapped-artifact")) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) } - badParent := filepath.Join(dir, "bad-parent") - if err := os.Symlink(realDir, badParent); err != nil { + beforeFragment, _, err := readCurrent(j.FragmentPath) + if err != nil { t.Fatal(err) } - if err := m.ConfigureLayout(realDir, filepath.Join(badParent, "meta.json")); err == nil { - t.Fatal("symlink sidecar root accepted") + called := false + if err := m.RequireRecoveredAuthorized(context.Background(), func(model.TaskResult) error { called = true; return nil }, "node-a", authority); err == nil || !strings.Contains(err.Error(), "changed after authority capture") { + t.Fatalf("error=%v", err) + } + afterFragment, _, err := readCurrent(j.FragmentPath) + if err != nil { + t.Fatal(err) + } + if called || !bytes.Equal(beforeFragment, afterFragment) { + t.Fatal("journal swap mutated result or artifact") } } +func TestRecoveryRejectsOversizeArtifactAndBackupSymlink(t *testing.T) { + t.Run("oversize current", func(t *testing.T) { + m, dir := testManager(t) + configDir := filepath.Join(dir, "conf") + mustMkdir(t, configDir) + if err := m.ConfigureLayout(configDir, filepath.Join(dir, "meta.json")); err != nil { + t.Fatal(err) + } + path := filepath.Join(m.configDir, "lattice-linechain-0123456789abcdef0123.json") + if err := os.WriteFile(path, bytes.Repeat([]byte("x"), maxArtifactSize+1), 0o600); err != nil { + t.Fatal(err) + } + if _, _, err := readCurrent(path); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("error=%v", err) + } + }) + t.Run("backup symlink", func(t *testing.T) { + m, path := appliedJournal(t, "task-backup", "lease") + j, err := readJournal(path) + if err != nil { + t.Fatal(err) + } + old := []byte("old") + j.FragmentHadOld = true + j.FragmentOld = digest(old) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + target := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(target, old, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path+".fragment.old"); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(j.FragmentPath) + if err != nil { + t.Fatal(err) + } + if err := m.RequireRecovered(context.Background(), func(model.TaskResult) error { return nil }, "node-a"); err == nil { + t.Fatal("symlink backup accepted") + } + after, err := os.ReadFile(j.FragmentPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("destination changed after rejected backup") + } + }) +} + +func TestApplyExactRetryReplaceAndRemove(t *testing.T) { + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + sidecarPath := filepath.Join(root, "meta.json") + if err := os.WriteFile(sidecarPath, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"keep"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil { + t.Fatal(err) + } + runs := 0 + m.run = func(context.Context, string, ...string) ([]byte, error) { runs++; return nil, nil } + basename := "lattice-linechain-0123456789abcdef0123.json" + fragment1, sidecar1 := "one", `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"one"}]}` + create := Document{Version: 2, Operation: "create", FragmentBasename: basename, Fragment: &fragment1, Sidecar: &sidecar1} + applyDoc(t, m, create, "create", "lease") + if err := m.Apply(context.Background(), bytes.NewReader(marshalDocument(t, create)), "create", "lease", digest([]byte("different-script"))); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("mismatched retry binding error=%v", err) + } + applyDoc(t, m, create, "create", "lease") + if runs != 6 { + t.Fatalf("exact retry reran host commands: %d", runs) + } + first, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "create", LeaseID: "lease"}, model.TaskResult{TaskID: "create", LeaseID: "lease", FinishedAt: time.Now().UTC()}) + if err != nil { + t.Fatal(err) + } + second, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "create", LeaseID: "lease"}, model.TaskResult{TaskID: "create", LeaseID: "lease", FinishedAt: time.Now().UTC().Add(time.Hour)}) + if err != nil || !second.FinishedAt.Equal(first.FinishedAt) { + t.Fatalf("exact retry result drift: first=%+v second=%+v err=%v", first, second, err) + } + if err := m.Cleanup("create", "lease"); err != nil { + t.Fatal(err) + } + + fragment2, sidecar2 := "two", `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"two"}]}` + replace := Document{Version: 2, Operation: "replace", FragmentBasename: basename, PreviousFragmentSHA256: digest([]byte(fragment1)), Fragment: &fragment2, Sidecar: &sidecar2} + applyDoc(t, m, replace, "replace", "lease") + resolveSuccess(t, m, "replace", "lease") + removeSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + remove := Document{Version: 2, Operation: "remove", FragmentBasename: basename, PreviousFragmentSHA256: digest([]byte(fragment2)), Sidecar: &removeSidecar} + applyDoc(t, m, remove, "remove", "lease") + resolveSuccess(t, m, "remove", "lease") + if _, err := os.Stat(filepath.Join(m.configDir, basename)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fragment remains: %v", err) + } + b, err := os.ReadFile(m.sidecarPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"ordinary":"keep"`) { + t.Fatalf("ordinary sidecar field lost: %s", b) + } +} + +func TestCommandAndRollbackFailureMatrix(t *testing.T) { + for _, failAt := range []string{"check", "restart", "active"} { + t.Run(failAt, func(t *testing.T) { + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + sidecarPath := filepath.Join(root, "meta.json") + oldFragment := "old" + oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"secret-not-logged"}` + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") + if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil { + t.Fatal(err) + } + m.run = func(_ context.Context, name string, _ ...string) ([]byte, error) { + if name == failAt { + return []byte("old new secret-not-logged private-key-canary"), errors.New("failed") + } + return nil, nil + } + newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + err := applyDocErr(m, d, "task-"+failAt, "lease") + if err == nil || strings.Contains(err.Error(), "secret-not-logged") || strings.Contains(err.Error(), "private-key-canary") { + t.Fatalf("failure=%v", err) + } + journalBytes, readErr := os.ReadFile(m.journalPath("task-"+failAt, "lease")) + if readErr == nil && (bytes.Contains(journalBytes, []byte("secret-not-logged")) || bytes.Contains(journalBytes, []byte("private-key-canary"))) { + t.Fatalf("journal leaked command output: %s", journalBytes) + } + assertFile(t, fragmentPath, oldFragment) + assertFile(t, sidecarPath, oldSidecar) + }) + } + + t.Run("rollback backup failure is loud", func(t *testing.T) { + m, path := appliedJournal(t, "task-rollback", "lease") + j, err := readJournal(path) + if err != nil { + t.Fatal(err) + } + j.Phase = "pair_published" + j.FragmentHadOld = true + j.FragmentOld = digest([]byte("old")) + if err := writeJSON(path, j); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(t.TempDir(), "missing"), path+".fragment.old"); err != nil { + t.Fatal(err) + } + if err := m.RequireRecovered(context.Background(), func(model.TaskResult) error { return nil }, "node-a"); err == nil { + t.Fatal("rollback failure was hidden") + } + }) +} + +func TestJournalAndPublishFailureMatrix(t *testing.T) { + for _, failAt := range []string{"journal", "fragment publish", "sidecar publish"} { + t.Run(failAt, func(t *testing.T) { + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") + sidecarPath := filepath.Join(root, "meta.json") + oldFragment := "old" + oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}` + if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + failed := false + m.ConfigureMutationForTest(func(path string, content *string) error { + shouldFail := (failAt == "fragment publish" && path == m.configDir+string(filepath.Separator)+filepath.Base(fragmentPath)) || + (failAt == "sidecar publish" && path == m.sidecarPath) + if shouldFail && !failed { + failed = true + return errors.New("injected publish failure") + } + return publish(path, content) + }, func(path string, value any) error { + if failAt == "journal" && !failed { + failed = true + return errors.New("injected temp journal failure") + } + return writeJSON(path, value) + }) + newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + err := applyDocErr(m, d, "task-"+failAt, "lease") + if err == nil || !failed { + t.Fatalf("injected failure not surfaced: %v", err) + } + assertFile(t, fragmentPath, oldFragment) + assertFile(t, sidecarPath, oldSidecar) + }) + } +} + +func TestPostRunAndRecoveryRuntimeVerificationFailures(t *testing.T) { + newAppliedReplace := func(t *testing.T) (*Manager, string, string, string) { + t.Helper() + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") + sidecarPath := filepath.Join(root, "meta.json") + oldFragment := "old" + oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}` + if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sidecarPath, []byte(oldSidecar), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("check", []string{"restart"}, []string{"active"}); err != nil { + t.Fatal(err) + } + m.run = func(context.Context, string, ...string) ([]byte, error) { return nil, nil } + newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + applyDoc(t, m, d, "task-post-run", "lease") + return m, fragmentPath, sidecarPath, oldSidecar + } + + for _, boundary := range []string{"resolve desired", "recover desired"} { + t.Run(boundary, func(t *testing.T) { + m, fragmentPath, sidecarPath, oldSidecar := newAppliedReplace(t) + failed := false + m.run = func(_ context.Context, name string, _ ...string) ([]byte, error) { + if name == "active" && !failed { + failed = true + return nil, errors.New("inactive") + } + return nil, nil + } + if boundary == "resolve desired" { + result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-post-run", LeaseID: "lease"}, model.TaskResult{TaskID: "task-post-run", LeaseID: "lease", FinishedAt: time.Now().UTC()}) + if err != nil || result.ExitCode == 0 || !strings.Contains(result.Error, "exact old pair restored") { + t.Fatalf("result=%+v err=%v", result, err) + } + } else { + var result model.TaskResult + if err := m.RequireRecovered(context.Background(), func(got model.TaskResult) error { result = got; return nil }, "node-a"); err != nil { + t.Fatal(err) + } + if result.ExitCode == 0 || !strings.Contains(result.Error, "exact old pair restored") { + t.Fatalf("result=%+v", result) + } + } + assertFile(t, fragmentPath, "old") + assertFile(t, sidecarPath, oldSidecar) + }) + } + + t.Run("runner failure with inactive restored runtime remains nonterminal", func(t *testing.T) { + m, fragmentPath, sidecarPath, oldSidecar := newAppliedReplace(t) + m.run = func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("inactive") } + result := model.TaskResult{TaskID: "task-post-run", LeaseID: "lease", ExitCode: -1, Error: "helper killed", FinishedAt: time.Now().UTC()} + if _, err := m.ResolveAfterRun(context.Background(), model.Task{ID: result.TaskID, LeaseID: result.LeaseID}, result); err == nil { + t.Fatal("inactive restored runtime was recorded terminal") + } + assertFile(t, fragmentPath, "old") + assertFile(t, sidecarPath, oldSidecar) + j, err := readJournal(m.journalPath(result.TaskID, result.LeaseID)) + if err != nil { + t.Fatal(err) + } + if journalTerminal(j.Phase) || j.Result != nil { + t.Fatalf("failure became terminal: %+v", j) + } + }) +} + +func resolveSuccess(t *testing.T, m *Manager, taskID, leaseID string) { + t.Helper() + result := model.TaskResult{TaskID: taskID, LeaseID: leaseID, FinishedAt: time.Now().UTC()} + result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, result) + if err != nil || result.ExitCode != 0 { + t.Fatalf("resolve result=%+v err=%v", result, err) + } + if err := m.Cleanup(taskID, leaseID); err != nil { + t.Fatal(err) + } +} + +func appliedJournal(t *testing.T, taskID, leaseID string) (*Manager, string) { + t.Helper() + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + if err := m.ConfigureLayout(configDir, filepath.Join(root, "meta.json")); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` + applyDoc(t, m, Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}, taskID, leaseID) + return m, m.journalPath(taskID, leaseID) +} + func testManager(t *testing.T) (*Manager, string) { t.Helper() dir := filepath.Join(t.TempDir(), "txn") @@ -300,13 +975,21 @@ func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) { } } func applyDocErr(m *Manager, d Document, task, lease string) error { + b := marshalDocument(nil, d) + return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease, digest([]byte(task))) +} + +func marshalDocument(t *testing.T, d Document) []byte { d = BindDocument(d) - b, _ := json.Marshal(wireDocumentV2{ + b, err := json.Marshal(wireDocumentV2{ Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, PreviousFragmentSHA256: d.PreviousFragmentSHA256, Fragment: d.Fragment, Sidecar: d.Sidecar, FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256, }) - return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease) + if err != nil && t != nil { + t.Fatal(err) + } + return b } func assertFile(t *testing.T, p, want string) { t.Helper() diff --git a/internal/linechain/read_nofollow_other.go b/internal/linechain/read_nofollow_other.go new file mode 100644 index 0000000..45d735c --- /dev/null +++ b/internal/linechain/read_nofollow_other.go @@ -0,0 +1,26 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris + +package linechain + +import ( + "fmt" + "os" +) + +func openNoFollow(path string) (*os.File, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + opened, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + visible, err := os.Lstat(path) + if err != nil || visible.Mode()&os.ModeSymlink != 0 || !os.SameFile(opened, visible) { + _ = f.Close() + return nil, fmt.Errorf("path changed or is a symbolic link") + } + return f, nil +} diff --git a/internal/linechain/read_nofollow_unix.go b/internal/linechain/read_nofollow_unix.go new file mode 100644 index 0000000..b542db2 --- /dev/null +++ b/internal/linechain/read_nofollow_unix.go @@ -0,0 +1,18 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package linechain + +import ( + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +func openNoFollow(path string) (*os.File, error) { + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + return nil, fmt.Errorf("open without following links: %w", err) + } + return os.NewFile(uintptr(fd), path), nil +} diff --git a/internal/taskexec/taskexec.go b/internal/taskexec/taskexec.go index 84d052d..3161ac5 100644 --- a/internal/taskexec/taskexec.go +++ b/internal/taskexec/taskexec.go @@ -3,6 +3,8 @@ package taskexec import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "os" @@ -297,7 +299,14 @@ func (r Runner) effectiveUID() int { return os.Geteuid() } -func (r Runner) Run(task model.Task) model.TaskResult { +func (r Runner) Run(task model.Task) model.TaskResult { return r.run(task, false) } + +// RunLinechain runs an already validated E3 lease with the private host-local +// linechain authority in its environment. Callers must select this path from +// durable protocol metadata, never from script contents. +func (r Runner) RunLinechain(task model.Task) model.TaskResult { return r.run(task, true) } + +func (r Runner) run(task model.Task, trustedLinechain bool) model.TaskResult { start := time.Now().UTC() result := model.TaskResult{ TaskID: task.ID, @@ -398,12 +407,16 @@ func (r Runner) Run(task model.Task) model.TaskResult { if filepath.IsAbs(r.AgentBinary) { cmd.Env = append(cmd.Env, "LATTICE_AGENT_BIN="+r.AgentBinary) } - if filepath.IsAbs(r.LinechainTxnDir) { + if trustedLinechain && filepath.IsAbs(r.LinechainTxnDir) { cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TXN_DIR="+r.LinechainTxnDir) } - if filepath.IsAbs(r.LinechainConfigDir) && filepath.IsAbs(r.LinechainSidecarPath) { + if trustedLinechain && filepath.IsAbs(r.LinechainConfigDir) && filepath.IsAbs(r.LinechainSidecarPath) { cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_CONFIG_DIR="+r.LinechainConfigDir, "LATTICE_LINECHAIN_SIDECAR_PATH="+r.LinechainSidecarPath) } + if trustedLinechain { + sum := sha256.Sum256([]byte(task.Script)) + cmd.Env = append(cmd.Env, "LATTICE_LINECHAIN_TASK_SCRIPT_SHA256="+hex.EncodeToString(sum[:])) + } var stdout, stderr cappedBuffer stdout.limit = limit diff --git a/internal/taskexec/taskexec_test.go b/internal/taskexec/taskexec_test.go index 9e169c7..23f3c29 100644 --- a/internal/taskexec/taskexec_test.go +++ b/internal/taskexec/taskexec_test.go @@ -1,6 +1,8 @@ package taskexec import ( + "crypto/sha256" + "encoding/hex" "os" "path/filepath" "runtime" @@ -201,6 +203,31 @@ func TestRunnerPropagatesAbsoluteAgentBinary(t *testing.T) { } } +func TestRunnerRestrictsLinechainEnvironmentToTrustedPath(t *testing.T) { + root := t.TempDir() + work := filepath.Join(root, "work") + if err := os.Mkdir(work, 0o700); err != nil { + t.Fatal(err) + } + r := Runner{ + AllowExec: true, AllowRoot: true, WorkdirRoot: work, + AgentBinary: "/absolute/lattice-agent", LinechainTxnDir: filepath.Join(root, "txn"), + LinechainConfigDir: filepath.Join(root, "conf"), LinechainSidecarPath: filepath.Join(root, "meta.json"), + } + task := model.Task{ID: "task-env", LeaseID: "lease-env", Interpreter: "sh", TimeoutSec: 10, OutputLimit: 4096, + Script: `printf '%s|%s|%s|%s|%s' "$LATTICE_AGENT_BIN" "$LATTICE_LINECHAIN_TXN_DIR" "$LATTICE_LINECHAIN_CONFIG_DIR" "$LATTICE_LINECHAIN_SIDECAR_PATH" "$LATTICE_LINECHAIN_TASK_SCRIPT_SHA256"`} + ordinary := r.Run(task) + if ordinary.ExitCode != 0 || ordinary.Stdout != "/absolute/lattice-agent||||" { + t.Fatalf("ordinary task environment leaked linechain authority: %+v", ordinary) + } + trusted := r.RunLinechain(task) + sum := sha256.Sum256([]byte(task.Script)) + want := strings.Join([]string{r.AgentBinary, r.LinechainTxnDir, r.LinechainConfigDir, r.LinechainSidecarPath, hex.EncodeToString(sum[:])}, "|") + if trusted.ExitCode != 0 || trusted.Stdout != want { + t.Fatalf("trusted E3 environment = %q result=%+v, want %q", trusted.Stdout, trusted, want) + } +} + func TestRunnerSetsPrivateTaskTempEnvironment(t *testing.T) { r := Runner{AllowExec: true, getUID: nonRootUID} result := r.Run(model.Task{ diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 8923ad6..2f2c568 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -13,11 +13,41 @@ printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || { } probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")" -trap 'find "$probe" -type f -delete 2>/dev/null || true; rmdir "$probe" 2>/dev/null || true' EXIT HUP INT TERM +e2e_root="$(mktemp -d "${TMPDIR:-/tmp}/lattice-linechain-e2e.XXXXXX")" +cleanup() { + status=$? + trap - EXIT HUP INT TERM + if ps -axo command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep >/dev/null; then + echo "linechain E2E leaked process for $e2e_root" >&2 + ps -axo pid=,ppid=,command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep >&2 || true + ps -axo pid=,command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep | while read -r leaked_pid _; do + kill -TERM "-$leaked_pid" 2>/dev/null || kill -TERM "$leaked_pid" 2>/dev/null || true + sleep 0.1 + kill -KILL "-$leaked_pid" 2>/dev/null || kill -KILL "$leaked_pid" 2>/dev/null || true + done + status=1 + fi + find "$probe" "$e2e_root" -type f -delete 2>/dev/null || true + find "$probe" "$e2e_root" -depth -type d -exec rmdir {} \; 2>/dev/null || true + exit "$status" +} +trap cleanup EXIT HUP INT TERM + +# Prove the root-tagged process detector before relying on it for the real run. +sh -c 'trap : TERM; while :; do sleep 1; done' "$e2e_root" & +detector_pid=$! +ps -axo command= | grep -F "$e2e_root" | grep -F 'while :' >/dev/null || { + kill "$detector_pid" 2>/dev/null || true + wait "$detector_pid" 2>/dev/null || true + echo "linechain E2E leak detector self-test failed" >&2 + exit 1 +} +kill -KILL "$detector_pid" 2>/dev/null || true +wait "$detector_pid" 2>/dev/null || true cat >"$probe/config.json" <<'JSON' {"inbounds":[],"outbounds":[{"type":"direct","tag":"direct"}],"route":{"final":"direct"}} JSON "$bin" check -C "$probe" >/dev/null 2>&1 || { echo "sing-box 1.13.x config check failed" >&2; exit 1; } printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')" -LATTICE_SINGBOX_E2E_BIN="$bin" go test -tags=linechain_e2e ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v +LATTICE_SINGBOX_E2E_BIN="$bin" LATTICE_LINECHAIN_E2E_ROOT="$e2e_root" go test -tags=linechain_e2e ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From 7a67ff9212dd05a13045ada0fef53554e1e7ba1c Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 08:21:11 -0700 Subject: [PATCH 24/43] Make taskexec helper tests honor Linux re-exec semantics The cmd package test binary now enters the same rlimit child shim as the production agent before testing starts. This prevents a trusted E3 shell test from recursively running the entire test suite when Runner re-execs /proc/self/exe on Linux. Constraint: Linux task execution re-execs the current binary before invoking the interpreter Rejected: Increase the helper timeout | it would mask recursive test execution and keep CI flaky Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep cmd TestMain aligned with production taskexec.MaybeRunChildShim behavior Tested: helper race test x10; go test -race -cover ./...; go test ./...; go vet ./...; exact sing-box 1.13.18 E2E with zero residual helpers; installer and release checks Not-tested: GitHub CI for this new commit --- cmd/lattice-agent/main_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index aa0f0c1..48334ca 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -22,6 +22,13 @@ import ( type roundTripFunc func(*http.Request) (*http.Response, error) +func TestMain(m *testing.M) { + if taskexec.MaybeRunChildShim(os.Args) { + os.Exit(0) + } + os.Exit(m.Run()) +} + func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } From 62449db6a9057676f36f7be2088d1fdb8fa2525a Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 08:46:34 -0700 Subject: [PATCH 25/43] Bind source-scoped sidecar patches without host-state authority drift Canonical semantic patches carry only the source identity and expected/desired downstream transition. The agent preserves unrelated metadata, applies an exact chain-object CAS, and validates the issued patch plus semantic artifact digests without treating whole host sidecar input or output as server-owned authority. Constraint: Strict alpha contract is linechain-e3-v2 with no compatibility branch Rejected: Bind the whole host sidecar base/output in approval | ordinary metadata is host-owned and may evolve independently Rejected: Replace the complete inbounds array | deletes unrelated ordinary-writer state Rejected: Recompute and overwrite issued digests | breaks approval and replay authority Confidence: high Scope-risk: narrow Directive: Keep merged output digests agent-journal-only and out of wire approval authority Tested: go test ./internal/linechain -count=1; git diff --check Not-tested: server-rendered fixture and Document/Apply/journal integration pending exact nullable-field ACK --- internal/linechain/sidecar_patch.go | 233 +++++++++++++++++++++++ internal/linechain/sidecar_patch_test.go | 120 ++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 internal/linechain/sidecar_patch.go create mode 100644 internal/linechain/sidecar_patch_test.go diff --git a/internal/linechain/sidecar_patch.go b/internal/linechain/sidecar_patch.go new file mode 100644 index 0000000..4d2560e --- /dev/null +++ b/internal/linechain/sidecar_patch.go @@ -0,0 +1,233 @@ +package linechain + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "strings" +) + +const semanticSidecarSchema = "lattice.singbox-metadata.v2" + +var lowercaseUUIDv4RE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + +type semanticSidecarPatchV2 struct { + Schema string `json:"schema"` + SourceLineUUID string `json:"source_line_uuid"` + SourceInboundTag string `json:"source_inbound_tag"` + ExpectedDownstreamLineUUID *string `json:"expected_downstream_line_uuid"` + DesiredDownstreamLineUUID *string `json:"desired_downstream_line_uuid"` +} + +type semanticSidecarBinding struct { + PatchSHA256 string + ArtifactSHA256 string +} + +func canonicalSemanticSidecarPatch(raw []byte) (semanticSidecarPatchV2, []byte, error) { + var fields map[string]json.RawMessage + if err := decodeJSONObject(raw, &fields); err != nil { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) + } + for _, name := range []string{"schema", "source_line_uuid", "source_inbound_tag", "expected_downstream_line_uuid", "desired_downstream_line_uuid"} { + if _, ok := fields[name]; !ok { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch field %s must be present", name) + } + } + if len(fields) != 5 { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch has unknown fields") + } + var patch semanticSidecarPatchV2 + if err := json.Unmarshal(raw, &patch); err != nil { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) + } + if patch.Schema != semanticSidecarSchema || !lowercaseUUIDv4RE.MatchString(patch.SourceLineUUID) || strings.TrimSpace(patch.SourceInboundTag) == "" || patch.SourceInboundTag != strings.TrimSpace(patch.SourceInboundTag) { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch identity is invalid") + } + for _, value := range []*string{patch.ExpectedDownstreamLineUUID, patch.DesiredDownstreamLineUUID} { + if value != nil && !lowercaseUUIDv4RE.MatchString(*value) { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar downstream identity is invalid") + } + } + canonical, err := json.Marshal(patch) + if err != nil { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("encode semantic sidecar patch: %w", err) + } + if !bytes.Equal(raw, canonical) { + return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch is not canonical") + } + return patch, canonical, nil +} + +func semanticArtifactDigest(fragment *string, patch []byte) string { + var raw []byte + if fragment != nil { + raw = append(raw, []byte(*fragment)...) + } + raw = append(raw, 0) + raw = append(raw, patch...) + return digest(raw) +} + +func verifySemanticSidecarBinding(fragment *string, patch []byte, issued semanticSidecarBinding) error { + checks := []struct{ label, want, got string }{ + {label: "patch", want: issued.PatchSHA256, got: digest(patch)}, + {label: "artifact", want: issued.ArtifactSHA256, got: semanticArtifactDigest(fragment, patch)}, + } + for _, check := range checks { + if !validSHA(check.want) || check.want != check.got { + return fmt.Errorf("semantic sidecar %s digest binding mismatch", check.label) + } + } + return nil +} + +// mergeManagedSidecar applies one authenticated source patch to an existing +// metadata v2 sidecar. It changes only the matched inbound's chain object. +func mergeManagedSidecar(current []byte, patch semanticSidecarPatchV2) ([]byte, error) { + var top map[string]json.RawMessage + if err := decodeJSONObject(current, &top); err != nil { + return nil, fmt.Errorf("decode current semantic sidecar: %w", err) + } + if err := validateSemanticSidecarTop(top); err != nil { + return nil, fmt.Errorf("current semantic sidecar: %w", err) + } + var inbounds []json.RawMessage + if err := json.Unmarshal(top["inbounds"], &inbounds); err != nil { + return nil, fmt.Errorf("decode current semantic sidecar inbounds: %w", err) + } + match := -1 + seenUUID := make(map[string]struct{}, len(inbounds)) + seenTag := make(map[string]struct{}, len(inbounds)) + for i, raw := range inbounds { + uuid, tag, err := semanticInboundIdentity(raw) + if err != nil { + return nil, fmt.Errorf("current semantic sidecar inbound %d: %w", i, err) + } + if _, ok := seenUUID[uuid]; ok { + return nil, fmt.Errorf("current semantic sidecar has duplicate line_uuid %q", uuid) + } + if _, ok := seenTag[tag]; ok { + return nil, fmt.Errorf("current semantic sidecar has duplicate tag %q", tag) + } + seenUUID[uuid], seenTag[tag] = struct{}{}, struct{}{} + if uuid == patch.SourceLineUUID || tag == patch.SourceInboundTag { + if uuid != patch.SourceLineUUID || tag != patch.SourceInboundTag || match >= 0 { + return nil, fmt.Errorf("current semantic sidecar source identity is ambiguous") + } + match = i + } + } + if match < 0 { + return nil, fmt.Errorf("current semantic sidecar source identity is missing") + } + var inbound map[string]json.RawMessage + if err := decodeJSONObject(inbounds[match], &inbound); err != nil { + return nil, err + } + currentDownstream, err := currentDownstreamIdentity(inbound) + if err != nil { + return nil, err + } + if !nullableStringEqual(currentDownstream, patch.ExpectedDownstreamLineUUID) { + return nil, fmt.Errorf("current semantic sidecar downstream identity changed") + } + if patch.DesiredDownstreamLineUUID == nil { + delete(inbound, "chain") + } else { + chain := struct { + DownstreamLineUUID string `json:"downstream_line_uuid"` + }{DownstreamLineUUID: *patch.DesiredDownstreamLineUUID} + encoded, err := json.Marshal(chain) + if err != nil { + return nil, err + } + inbound["chain"] = encoded + } + encodedInbound, err := json.Marshal(inbound) + if err != nil { + return nil, err + } + inbounds[match] = encodedInbound + encodedInbounds, err := json.Marshal(inbounds) + if err != nil { + return nil, err + } + top["inbounds"] = encodedInbounds + out, err := json.Marshal(top) + if err != nil { + return nil, err + } + return append(out, '\n'), nil +} + +func currentDownstreamIdentity(inbound map[string]json.RawMessage) (*string, error) { + raw, ok := inbound["chain"] + if !ok { + return nil, nil + } + var chain map[string]json.RawMessage + if err := decodeJSONObject(raw, &chain); err != nil { + return nil, fmt.Errorf("current semantic sidecar chain: %w", err) + } + rawDownstream, ok := chain["downstream_line_uuid"] + if !ok { + return nil, fmt.Errorf("current semantic sidecar chain downstream_line_uuid must be present") + } + if bytes.Equal(bytes.TrimSpace(rawDownstream), []byte("null")) { + return nil, nil + } + var value string + if err := json.Unmarshal(rawDownstream, &value); err != nil || !lowercaseUUIDv4RE.MatchString(value) { + return nil, fmt.Errorf("current semantic sidecar downstream_line_uuid is invalid") + } + return &value, nil +} + +func nullableStringEqual(a, b *string) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func validateSemanticSidecarTop(top map[string]json.RawMessage) error { + var schema string + if err := json.Unmarshal(top["schema"], &schema); err != nil || schema != semanticSidecarSchema { + return fmt.Errorf("unsupported schema") + } + var inbounds []json.RawMessage + if err := json.Unmarshal(top["inbounds"], &inbounds); err != nil || inbounds == nil { + return fmt.Errorf("inbounds must be an array") + } + return nil +} + +func semanticInboundIdentity(raw json.RawMessage) (string, string, error) { + var value map[string]json.RawMessage + if err := decodeJSONObject(raw, &value); err != nil { + return "", "", err + } + var uuid, tag string + if err := json.Unmarshal(value["line_uuid"], &uuid); err != nil || !lowercaseUUIDv4RE.MatchString(uuid) { + return "", "", fmt.Errorf("line_uuid must be a lowercase UUIDv4 string") + } + if err := json.Unmarshal(value["tag"], &tag); err != nil || strings.TrimSpace(tag) == "" || tag != strings.TrimSpace(tag) { + return "", "", fmt.Errorf("tag must be a non-empty canonical string") + } + return uuid, tag, nil +} + +func decodeJSONObject(raw []byte, dst *map[string]json.RawMessage) error { + if len(bytes.TrimSpace(raw)) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("must be a non-null object") + } + if err := json.Unmarshal(raw, dst); err != nil { + return err + } + if *dst == nil { + return fmt.Errorf("must be a non-null object") + } + return nil +} diff --git a/internal/linechain/sidecar_patch_test.go b/internal/linechain/sidecar_patch_test.go new file mode 100644 index 0000000..6cf48a6 --- /dev/null +++ b/internal/linechain/sidecar_patch_test.go @@ -0,0 +1,120 @@ +package linechain + +import ( + "encoding/json" + "strings" + "testing" +) + +const ( + sourceUUID = "11111111-1111-4111-8111-1111111111aa" + oldUUID = "22222222-2222-4222-8222-222222222222" + newUUID = "33333333-3333-4333-8333-333333333333" + otherUUID = "44444444-4444-4444-8444-444444444444" +) + +func stringPtr(value string) *string { return &value } + +func patchBytes(expected, desired *string) []byte { + b, err := json.Marshal(semanticSidecarPatchV2{Schema: semanticSidecarSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", ExpectedDownstreamLineUUID: expected, DesiredDownstreamLineUUID: desired}) + if err != nil { + panic(err) + } + return b +} + +func TestCanonicalSemanticSidecarPatchRequiresExactShape(t *testing.T) { + raw := patchBytes(nil, stringPtr(newUUID)) + patch, canonical, err := canonicalSemanticSidecarPatch(raw) + if err != nil { + t.Fatal(err) + } + if patch.SourceLineUUID != sourceUUID || string(canonical) != string(raw) { + t.Fatalf("canonical patch mismatch: %s", canonical) + } + for _, invalid := range [][]byte{ + []byte(`{"schema":"lattice.singbox-metadata.v2","source_line_uuid":"` + sourceUUID + `","source_inbound_tag":"source","desired_downstream_line_uuid":null}`), + append(append([]byte(nil), raw...), '\n'), + []byte(strings.Replace(string(raw), sourceUUID, strings.ToUpper(sourceUUID), 1)), + } { + if _, _, err := canonicalSemanticSidecarPatch(invalid); err == nil { + t.Fatalf("invalid patch accepted: %s", invalid) + } + } +} + +func TestMergeManagedSidecarPreservesUnrelatedStateAndInboundFields(t *testing.T) { + current := []byte(`{"schema":"lattice.singbox-metadata.v2","writer":"ordinary","unknown":{"keep":true},"inbounds":[{"tag":"other","line_uuid":"` + otherUUID + `","ordinary":"keep"},{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":"` + oldUUID + `","discard":"old"},"local":"preserve"}]}`) + patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), stringPtr(newUUID))) + out, err := mergeManagedSidecar(current, patch) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatal(err) + } + if got["writer"] != "ordinary" || got["unknown"].(map[string]any)["keep"] != true { + t.Fatalf("top-level state lost: %s", out) + } + inbounds := got["inbounds"].([]any) + if len(inbounds) != 2 || inbounds[0].(map[string]any)["ordinary"] != "keep" || inbounds[1].(map[string]any)["local"] != "preserve" { + t.Fatalf("unrelated state changed: %s", out) + } + chain := inbounds[1].(map[string]any)["chain"].(map[string]any) + if chain["downstream_line_uuid"] != newUUID || len(chain) != 1 { + t.Fatalf("chain was not replaced exactly: %s", out) + } +} + +func TestMergeManagedSidecarRemoveDeletesOnlyChain(t *testing.T) { + current := []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":"` + oldUUID + `"},"keep":true}]}`) + patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), nil)) + out, err := mergeManagedSidecar(current, patch) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "chain") || !strings.Contains(string(out), `"keep":true`) { + t.Fatalf("remove changed more than chain: %s", out) + } +} + +func TestMergeManagedSidecarRejectsIdentityAndBaseDrift(t *testing.T) { + patch, _, _ := canonicalSemanticSidecarPatch(patchBytes(stringPtr(oldUUID), stringPtr(newUUID))) + cases := []string{ + `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + otherUUID + `"}]}`, + `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"one","line_uuid":"` + sourceUUID + `"},{"tag":"two","line_uuid":"` + sourceUUID + `"}]}`, + `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + sourceUUID + `","chain":{"downstream_line_uuid":null}}]}`, + `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"other","line_uuid":"` + otherUUID + `"}]}`, + } + for _, current := range cases { + if _, err := mergeManagedSidecar([]byte(current), patch); err == nil { + t.Fatalf("invalid identity/base accepted: %s", current) + } + } +} + +func TestSemanticBindingNeverRewritesIssuedDigests(t *testing.T) { + fragment := `{"outbounds":[]}` + patch := patchBytes(nil, stringPtr(newUUID)) + issued := semanticSidecarBinding{PatchSHA256: digest(patch), ArtifactSHA256: semanticArtifactDigest(&fragment, patch)} + if err := verifySemanticSidecarBinding(&fragment, patch, issued); err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(*semanticSidecarBinding){ + "patch": func(v *semanticSidecarBinding) { v.PatchSHA256 = digest([]byte("other")) }, + "artifact": func(v *semanticSidecarBinding) { v.ArtifactSHA256 = digest([]byte("other")) }, + } { + t.Run(name, func(t *testing.T) { + mismatch := issued + mutate(&mismatch) + before := mismatch + if err := verifySemanticSidecarBinding(&fragment, patch, mismatch); err == nil { + t.Fatal("mismatch accepted") + } + if mismatch != before { + t.Fatalf("issued digest rewritten: %+v -> %+v", before, mismatch) + } + }) + } +} From c063de73db11f5420739b131a254360252bdf245 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 09:19:58 -0700 Subject: [PATCH 26/43] Make line-chain approval survive host-local semantic merging Replace whole-sidecar byte authority with a canonical source-scoped patch and semantic artifact binding, while retaining actual merged output digests only in the crash journal. Recovery now cross-checks the independently issued artifact embedded in the exact server task wrapper, rejects legacy durable entries, and preserves unrelated host metadata through UUID-plus-tag CAS updates. Constraint: Server approval may bind only canonical PatchSHA and ArtifactSHA, never mutable host base or output bytes Constraint: Existing metadata v2 sidecars remain owned by ordinary host writers outside the matched chain member Rejected: Hash the merged host sidecar as issued authority | host-local unrelated fields make that value unavailable to the server before execution Rejected: Trust journal ArtifactSHA during recovery | that makes authority circular after local journal corruption Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep the v2 wrapper, patch, artifact, journal output, and outbox authority fields distinct; do not restore raw whole-sidecar binding Tested: production server fixture and fixed vectors; focused semantic, recovery, outbox, and taskexec tests; go test ./...; go test -race -cover ./...; go vet ./...; installer, release-workflow, proto, install/version/compat, and secret gates Not-tested: Full real sing-box lifecycle E2E is owned by Task 18 --- cmd/lattice-agent/main.go | 88 +++++++- cmd/lattice-agent/main_test.go | 134 ++++++++++-- internal/linechain/manager.go | 249 +++++++++++---------- internal/linechain/manager_test.go | 261 ++++++++++++++++++----- internal/linechain/sidecar_patch.go | 102 ++++++--- internal/linechain/sidecar_patch_test.go | 74 ++++++- internal/taskoutbox/outbox.go | 5 +- internal/taskoutbox/outbox_test.go | 18 ++ 8 files changed, 688 insertions(+), 243 deletions(-) diff --git a/cmd/lattice-agent/main.go b/cmd/lattice-agent/main.go index 9575209..673da3d 100644 --- a/cmd/lattice-agent/main.go +++ b/cmd/lattice-agent/main.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -1226,7 +1227,7 @@ func runTasks(cfg agentConfig, runner taskRunner, outbox taskResultOutbox, manag if err := validateDurablePair(leased, manager, cfg.LinechainReady); err != nil { return err } - if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v1" && leased.DurableProtocol != "netguard-v1" { + if leased.DurableProtocol != "" && leased.DurableProtocol != "linechain-e3-v2" && leased.DurableProtocol != "netguard-v1" { return fmt.Errorf("unsupported durable protocol %q", leased.DurableProtocol) } if !leased.DurableResult { @@ -1366,21 +1367,25 @@ func captureLinechainAuthority(manager *linechain.Manager, outbox taskResultOutb for _, entry := range entries { key := entry.Task.ID + "\x00" + entry.Task.LeaseID ref, hasJournal := journals[key] - if hasJournal && entry.DurableProtocol != "linechain-e3-v1" { + if hasJournal && entry.DurableProtocol != "linechain-e3-v2" { return nil, fmt.Errorf("linechain journal %s has mismatched outbox protocol", entry.Task.ID) } - if hasJournal && entry.DurableProtocol == "linechain-e3-v1" { + if hasJournal && entry.DurableProtocol == "linechain-e3-v2" { if ref.TaskScriptSHA != linechainTaskScriptSHA(entry.Task.Script) { return nil, fmt.Errorf("linechain journal %s does not match the exact outbox task script", entry.Task.ID) } + artifactSHA, err := linechainArtifactSHAFromTaskScript(entry.Task.Script) + if err != nil || artifactSHA != ref.ArtifactSHA256 { + return nil, fmt.Errorf("linechain journal %s does not match the issued artifact in the exact outbox script", entry.Task.ID) + } matchedE3[key] = struct{}{} } - if entry.State == "leased" && entry.DurableProtocol == "linechain-e3-v1" { + if entry.State == "leased" && entry.DurableProtocol == "linechain-e3-v2" { if !hasJournal { return nil, fmt.Errorf("leased E3 outbox %s lacks exact linechain journal", entry.Task.ID) } } - if entry.State == "completed" && entry.DurableProtocol == "linechain-e3-v1" && hasJournal { + if entry.State == "completed" && entry.DurableProtocol == "linechain-e3-v2" && hasJournal { if !ref.Terminal || entry.Result == nil || ref.Result == nil || !sameTaskResult(*entry.Result, *ref.Result) { return nil, fmt.Errorf("completed E3 outbox %s conflicts with linechain terminal result", entry.Task.ID) } @@ -1393,7 +1398,7 @@ func captureLinechainAuthority(manager *linechain.Manager, outbox taskResultOutb } authority := make(map[string]linechain.RecoveryAuthority, len(journals)) for key, ref := range journals { - authority[key] = linechain.RecoveryAuthority{TaskScriptSHA: ref.TaskScriptSHA, CombinedSHA256: ref.CombinedSHA256, Phase: ref.Phase, ResultSHA256: taskResultPointerSHA(ref.Result), JournalSHA256: ref.JournalSHA256} + authority[key] = linechain.RecoveryAuthority{TaskScriptSHA: ref.TaskScriptSHA, ArtifactSHA256: ref.ArtifactSHA256, Phase: ref.Phase, ResultSHA256: taskResultPointerSHA(ref.Result), JournalSHA256: ref.JournalSHA256} } return authority, nil } @@ -1409,6 +1414,69 @@ func linechainTaskScriptSHA(script string) string { return hex.EncodeToString(sum[:]) } +func linechainArtifactSHAFromTaskScript(script string) (string, error) { + const prefix = "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '" + const suffix = "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n" + if !strings.HasPrefix(script, prefix) || !strings.HasSuffix(script, suffix) { + return "", fmt.Errorf("linechain task script is not the canonical v2 wrapper") + } + encoded := strings.TrimSuffix(strings.TrimPrefix(script, prefix), suffix) + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", fmt.Errorf("decode linechain task document: %w", err) + } + fields, err := decodeUniqueTopLevelFields(raw) + if err != nil { + return "", fmt.Errorf("decode linechain task document: %w", err) + } + var version int + var protocol, artifact string + if err := json.Unmarshal(fields["version"], &version); err != nil || version != 2 { + return "", fmt.Errorf("linechain task document version is invalid") + } + if err := json.Unmarshal(fields["durable_protocol"], &protocol); err != nil || protocol != "linechain-e3-v2" { + return "", fmt.Errorf("linechain task document protocol is invalid") + } + if err := json.Unmarshal(fields["artifact_sha256"], &artifact); err != nil || !guardManagedSHARe.MatchString(artifact) { + return "", fmt.Errorf("linechain task document artifact is invalid") + } + return artifact, nil +} + +func decodeUniqueTopLevelFields(raw []byte) (map[string]json.RawMessage, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + token, err := dec.Token() + if err != nil || token != json.Delim('{') { + return nil, fmt.Errorf("document must be an object") + } + fields := make(map[string]json.RawMessage) + for dec.More() { + token, err := dec.Token() + if err != nil { + return nil, err + } + name, ok := token.(string) + if !ok { + return nil, fmt.Errorf("document field name is invalid") + } + if _, exists := fields[name]; exists { + return nil, fmt.Errorf("duplicate document field %s", name) + } + var value json.RawMessage + if err := dec.Decode(&value); err != nil { + return nil, err + } + fields[name] = value + } + if _, err := dec.Token(); err != nil { + return nil, err + } + if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("unexpected trailing data") + } + return fields, nil +} + func taskResultPointerSHA(result *model.TaskResult) string { if result == nil { return "" @@ -1428,17 +1496,17 @@ func validateDurablePair(task leasedAgentTask, manager *linechain.Manager, ready } return nil } - if task.DurableProtocol != "linechain-e3-v1" && task.DurableProtocol != "netguard-v1" { + if task.DurableProtocol != "linechain-e3-v2" && task.DurableProtocol != "netguard-v1" { return fmt.Errorf("durable task has unsupported protocol %q", task.DurableProtocol) } - if task.DurableProtocol == "linechain-e3-v1" && (manager == nil || !ready) { + if task.DurableProtocol == "linechain-e3-v2" && (manager == nil || !ready) { return fmt.Errorf("linechain task received while durable linechain is unavailable") } return nil } func isLinechainTask(task leasedAgentTask) bool { - return task.DurableResult && task.DurableProtocol == "linechain-e3-v1" + return task.DurableResult && task.DurableProtocol == "linechain-e3-v2" } func requireLinechainRecovered(ctx context.Context, manager *linechain.Manager, outbox taskResultOutbox, nodeID string) error { @@ -1481,7 +1549,7 @@ func flushTaskResultsRetain(cfg agentConfig, outbox taskResultOutbox, retain boo // Pending entries predate typed lease metadata; retain only the exact // helper namespace marker during cleanup recovery. Live classification is // always driven by durable_protocol on the leased response. - if retain && entry.DurableProtocol == "linechain-e3-v1" { + if retain && entry.DurableProtocol == "linechain-e3-v2" { continue } if err := outbox.Remove(entry); err != nil { diff --git a/cmd/lattice-agent/main_test.go b/cmd/lattice-agent/main_test.go index 48334ca..398acd8 100644 --- a/cmd/lattice-agent/main_test.go +++ b/cmd/lattice-agent/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "io" @@ -374,19 +375,19 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } + writeLinechainSourceSidecar(t, filepath.Join(root, "lattice-metadata.json")) fragmentPath := filepath.Join(conf, "lattice-linechain-0123456789abcdef0123.json") fragment := "{\"outbounds\":[]}" - sidecar := "{\"schema\":\"lattice.singbox-metadata.v2\",\"inbounds\":[]}" - docValue := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, Sidecar: &sidecar}) + docValue := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, SidecarPatch: testLinechainPatch("create")}) doc, err := json.Marshal(map[string]any{ - "version": docValue.Version, "operation": docValue.Operation, "fragment_basename": docValue.FragmentBasename, - "fragment": docValue.Fragment, "sidecar": docValue.Sidecar, "fragment_sha256": docValue.FragmentSHA256, - "sidecar_sha256": docValue.SidecarSHA256, "combined_sha256": docValue.CombinedSHA256, + "version": docValue.Version, "durable_protocol": docValue.DurableProtocol, "operation": docValue.Operation, "fragment_basename": docValue.FragmentBasename, + "fragment": docValue.Fragment, "sidecar_patch": docValue.SidecarPatch, "previous_fragment_sha256": docValue.PreviousFragmentSHA256, + "fragment_sha256": docValue.FragmentSHA256, "sidecar_patch_sha256": docValue.SidecarPatchSHA256, "artifact_sha256": docValue.ArtifactSHA256, }) if err != nil { t.Fatal(err) } - task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: "# lattice-linechain-e3-v1\n\"$LATTICE_LINECHAIN_TXN_DIR\"=1 \"$LATTICE_AGENT_BIN\" -linechain-apply", TimeoutSec: 10, OutputLimit: 1024} + task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: testLinechainApplyScript(doc), TimeoutSec: 10, OutputLimit: 1024} genericTask := model.Task{ID: "task-generic", LeaseID: "lease-generic", Interpreter: "sh", Script: "echo generic"} netguardTask := model.Task{ID: "task-netguard", LeaseID: "lease-netguard", Interpreter: "sh", Script: "echo netguard"} runner := &linechainTaskRunner{manager: manager, doc: doc, linechainTaskID: task.ID, afterApply: func() { @@ -408,7 +409,7 @@ func TestRunTasksLinechainCompletesHandoffWithoutReplay(t *testing.T) { b, _ := json.Marshal([]leasedAgentTask{ {Task: genericTask}, {Task: netguardTask, DurableResult: true, DurableProtocol: "netguard-v1"}, - {Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v1"}, + {Task: task, DurableResult: true, DurableProtocol: "linechain-e3-v2"}, }) return testResponse(http.StatusOK, string(b)), nil } @@ -471,11 +472,21 @@ func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { t.Cleanup(func() { _ = outbox.Close() }) return manager, outbox } - task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: "true"} + fragment := `{}` + doc := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")}) + rawDoc, err := json.Marshal(map[string]any{ + "version": doc.Version, "durable_protocol": doc.DurableProtocol, "operation": doc.Operation, "fragment_basename": doc.FragmentBasename, + "fragment": doc.Fragment, "sidecar_patch": doc.SidecarPatch, "previous_fragment_sha256": doc.PreviousFragmentSHA256, + "fragment_sha256": doc.FragmentSHA256, "sidecar_patch_sha256": doc.SidecarPatchSHA256, "artifact_sha256": doc.ArtifactSHA256, + }) + if err != nil { + t.Fatal(err) + } + task := model.Task{ID: "task-chain", LeaseID: "lease-chain", Interpreter: "sh", Script: testLinechainApplyScript(rawDoc)} t.Run("leased E3 without journal blocks require gate", func(t *testing.T) { manager, outbox := newPair(t) - if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { t.Fatal(err) } if err := requireLinechainRecovered(context.Background(), manager, outbox, "node-a"); err == nil || !strings.Contains(err.Error(), "lacks exact linechain journal") { @@ -485,7 +496,7 @@ func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { t.Run("completed E3 without journal is allowed", func(t *testing.T) { manager, outbox := newPair(t) - if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { t.Fatal(err) } if _, err := outbox.Complete(model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}); err != nil { @@ -517,7 +528,7 @@ func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { t.Run("same IDs with different script are blocked", func(t *testing.T) { manager, outbox := newPair(t) - if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { t.Fatal(err) } other := task @@ -528,11 +539,59 @@ func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { } }) + t.Run("journal artifact must match exact issued script", func(t *testing.T) { + root := t.TempDir() + txnDir := filepath.Join(root, "txn") + manager, err := linechain.Open(txnDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = manager.Close() }) + outbox, err := taskoutbox.Open(filepath.Join(root, "outbox")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = outbox.Close() }) + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { + t.Fatal(err) + } + createLinechainJournal(t, manager, task) + entries, err := os.ReadDir(txnDir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := filepath.Join(txnDir, entry.Name()) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var journal map[string]any + if err := json.Unmarshal(raw, &journal); err != nil { + t.Fatal(err) + } + journal["artifact_sha256"] = strings.Repeat("a", 64) + tampered, err := json.Marshal(journal) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, tampered, 0o600); err != nil { + t.Fatal(err) + } + } + if err := crossCheckLinechainAuthority(manager, outbox); err == nil || !strings.Contains(err.Error(), "issued artifact") { + t.Fatalf("tampered journal artifact error = %v", err) + } + }) + t.Run("completed E3 requires exact terminal result", func(t *testing.T) { for _, mismatch := range []bool{false, true} { t.Run(map[bool]string{false: "matching", true: "mismatched"}[mismatch], func(t *testing.T) { manager, outbox := newPair(t) - if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { t.Fatal(err) } createLinechainJournal(t, manager, task) @@ -561,7 +620,7 @@ func TestLinechainAuthorityCrossCheckIsBidirectional(t *testing.T) { t.Run("completed E3 conflicts with nonterminal journal", func(t *testing.T) { manager, outbox := newPair(t) - if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { t.Fatal(err) } createLinechainJournal(t, manager, task) @@ -587,12 +646,13 @@ func createLinechainJournal(t *testing.T, manager *linechain.Manager, task model if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) + writeLinechainSourceSidecar(t, filepath.Join(root, "lattice-metadata.json")) + fragment := `{}` + d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")}) raw, err := json.Marshal(map[string]any{ - "version": d.Version, "operation": d.Operation, "fragment_basename": d.FragmentBasename, - "fragment": d.Fragment, "sidecar": d.Sidecar, "fragment_sha256": d.FragmentSHA256, - "sidecar_sha256": d.SidecarSHA256, "combined_sha256": d.CombinedSHA256, + "version": d.Version, "durable_protocol": d.DurableProtocol, "operation": d.Operation, "fragment_basename": d.FragmentBasename, + "fragment": d.Fragment, "sidecar_patch": d.SidecarPatch, "previous_fragment_sha256": d.PreviousFragmentSHA256, + "fragment_sha256": d.FragmentSHA256, "sidecar_patch_sha256": d.SidecarPatchSHA256, "artifact_sha256": d.ArtifactSHA256, }) if err != nil { t.Fatal(err) @@ -624,12 +684,13 @@ func TestTaskexecRunsRealLinechainHelperShell(t *testing.T) { if err := manager.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - fragment, sidecar := `{"outbounds":[]}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) + writeLinechainSourceSidecar(t, sidecarPath) + fragment := `{"outbounds":[]}` + d := linechain.BindDocument(linechain.Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testLinechainPatch("create")}) raw, err := json.Marshal(map[string]any{ - "version": d.Version, "operation": d.Operation, "fragment_basename": d.FragmentBasename, - "fragment": d.Fragment, "sidecar": d.Sidecar, "fragment_sha256": d.FragmentSHA256, - "sidecar_sha256": d.SidecarSHA256, "combined_sha256": d.CombinedSHA256, + "version": d.Version, "durable_protocol": d.DurableProtocol, "operation": d.Operation, "fragment_basename": d.FragmentBasename, + "fragment": d.Fragment, "sidecar_patch": d.SidecarPatch, "previous_fragment_sha256": d.PreviousFragmentSHA256, + "fragment_sha256": d.FragmentSHA256, "sidecar_patch_sha256": d.SidecarPatchSHA256, "artifact_sha256": d.ArtifactSHA256, }) if err != nil { t.Fatal(err) @@ -679,6 +740,33 @@ func TestTaskexecLinechainHelperProcess(t *testing.T) { } } +func writeLinechainSourceSidecar(t *testing.T, path string) { + t.Helper() + const sidecar = `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"11111111-1111-4111-8111-1111111111aa"}]}` + if err := os.WriteFile(path, []byte(sidecar), 0o600); err != nil { + t.Fatal(err) + } +} + +func testLinechainPatch(operation string) linechain.SidecarPatchV2 { + desired := "33333333-3333-4333-8333-333333333333" + patch := linechain.SidecarPatchV2{ + Schema: "lattice.singbox-linechain-sidecar-patch.v1", SourceLineUUID: "11111111-1111-4111-8111-1111111111aa", + SourceInboundTag: "source", DesiredDownstreamLineUUID: &desired, + } + if operation != "create" { + patch.ExpectedDownstreamLineUUID = &desired + } + if operation == "remove" { + patch.DesiredDownstreamLineUUID = nil + } + return patch +} + +func testLinechainApplyScript(document []byte) string { + return "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '" + base64.StdEncoding.EncodeToString(document) + "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n" +} + func TestRunTasksRestartFlushesCompletedResultBeforeFetch(t *testing.T) { oldClient := httpClient defer func() { httpClient = oldClient }() diff --git a/internal/linechain/manager.go b/internal/linechain/manager.go index bb41171..42270b8 100644 --- a/internal/linechain/manager.go +++ b/internal/linechain/manager.go @@ -22,7 +22,7 @@ import ( ) const ( - journalVersion = 1 + journalVersion = 2 maxDocumentSize = 4 << 20 maxArtifactSize = maxDocumentSize ) @@ -32,67 +32,78 @@ var linechainBasenameRE = regexp.MustCompile(`^lattice-linechain-[0-9a-f]{20}\.j // Document is the bounded server-rendered input consumed by -linechain-apply. // Desired content is never copied into the journal; only its digest is stored. type Document struct { - Version int `json:"version"` - Operation string `json:"operation"` - ConfigDir string `json:"-"` - FragmentBasename string `json:"fragment_basename,omitempty"` - FragmentPath string `json:"-"` - SidecarPath string `json:"-"` - PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` - Fragment *string `json:"fragment,omitempty"` - Sidecar *string `json:"sidecar,omitempty"` - FragmentSHA256 string `json:"fragment_sha256,omitempty"` - SidecarSHA256 string `json:"sidecar_sha256,omitempty"` - CombinedSHA256 string `json:"combined_sha256"` + Version int `json:"version"` + DurableProtocol string `json:"durable_protocol"` + Operation string `json:"operation"` + ConfigDir string `json:"-"` + FragmentBasename string `json:"fragment_basename"` + FragmentPath string `json:"-"` + SidecarPath string `json:"-"` + Fragment *string `json:"fragment"` + SidecarPatch SidecarPatchV2 `json:"sidecar_patch"` + PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"` + FragmentSHA256 *string `json:"fragment_sha256"` + SidecarPatchSHA256 string `json:"sidecar_patch_sha256"` + ArtifactSHA256 string `json:"artifact_sha256"` + SidecarPatchCanonical []byte `json:"-"` + SidecarOutput *string `json:"-"` } // wireDocumentV2 is the only server-controlled shape accepted by Apply. // Host paths and the ordinary-writer sidecar predecessor are intentionally not // representable on the wire; Apply derives paths from the local layout. type wireDocumentV2 struct { - Version int `json:"version"` - Operation string `json:"operation"` - FragmentBasename string `json:"fragment_basename"` - PreviousFragmentSHA256 string `json:"previous_fragment_sha256,omitempty"` - Fragment *string `json:"fragment,omitempty"` - Sidecar *string `json:"sidecar,omitempty"` - FragmentSHA256 string `json:"fragment_sha256,omitempty"` - SidecarSHA256 string `json:"sidecar_sha256,omitempty"` - CombinedSHA256 string `json:"combined_sha256"` + Version int `json:"version"` + DurableProtocol string `json:"durable_protocol"` + Operation string `json:"operation"` + FragmentBasename string `json:"fragment_basename"` + Fragment *string `json:"fragment"` + SidecarPatch SidecarPatchV2 `json:"sidecar_patch"` + PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"` + FragmentSHA256 *string `json:"fragment_sha256"` + SidecarPatchSHA256 string `json:"sidecar_patch_sha256"` + ArtifactSHA256 string `json:"artifact_sha256"` } -// BindDocument computes the deterministic desired artifact binding. +// BindDocument computes canonical test/server fixture bindings. Apply validates +// issued bindings and never calls this after reading or merging host state. func BindDocument(d Document) Document { - d.FragmentSHA256 = digestPtr(d.Fragment) - d.SidecarSHA256 = digestPtr(d.Sidecar) - var pair []byte - if d.Fragment != nil { - pair = append(pair, []byte(*d.Fragment)...) + if d.DurableProtocol == "" { + d.DurableProtocol = "linechain-e3-v2" } - pair = append(pair, 0) - if d.Sidecar != nil { - pair = append(pair, []byte(*d.Sidecar)...) - } - d.CombinedSHA256 = digest(pair) + if d.Fragment == nil { + d.FragmentSHA256 = nil + } else { + value := digest([]byte(*d.Fragment)) + d.FragmentSHA256 = &value + } + patch, _ := json.Marshal(d.SidecarPatch) + d.SidecarPatchCanonical = patch + d.SidecarPatchSHA256 = digest(patch) + binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: d.Operation, FragmentBasename: d.FragmentBasename, + PreviousFragmentSHA256: d.PreviousFragmentSHA256, FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256} + canonical, _ := canonicalSemanticArtifactBinding(binding) + d.ArtifactSHA256 = digest(canonical) return d } type journal struct { - Version int `json:"version"` - TaskID string `json:"task_id"` - LeaseID string `json:"lease_id"` - FragmentPath string `json:"fragment_path"` - SidecarPath string `json:"sidecar_path"` - FragmentOld string `json:"fragment_old_sha256,omitempty"` - SidecarOld string `json:"sidecar_old_sha256,omitempty"` - FragmentNew string `json:"fragment_desired_sha256,omitempty"` - SidecarNew string `json:"sidecar_desired_sha256,omitempty"` - CombinedNew string `json:"combined_desired_sha256"` - TaskScriptSHA string `json:"task_script_sha256"` - FragmentHadOld bool `json:"fragment_had_old"` - SidecarHadOld bool `json:"sidecar_had_old"` - Phase string `json:"phase"` - Result *model.TaskResult `json:"result,omitempty"` + Version int `json:"version"` + TaskID string `json:"task_id"` + LeaseID string `json:"lease_id"` + FragmentPath string `json:"fragment_path"` + SidecarPath string `json:"sidecar_path"` + FragmentOld string `json:"fragment_old_sha256,omitempty"` + SidecarOld string `json:"sidecar_old_sha256,omitempty"` + ArtifactSHA256 string `json:"artifact_sha256"` + SidecarPatchSHA256 string `json:"sidecar_patch_sha256"` + FragmentOutputSHA256 string `json:"fragment_output_sha256,omitempty"` + SidecarOutputSHA256 string `json:"sidecar_output_sha256"` + TaskScriptSHA string `json:"task_script_sha256"` + FragmentHadOld bool `json:"fragment_had_old"` + SidecarHadOld bool `json:"sidecar_had_old"` + Phase string `json:"phase"` + Result *model.TaskResult `json:"result,omitempty"` } type Manager struct { @@ -317,14 +328,36 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskS if wire.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", wire.Version) } + if wire.DurableProtocol != "linechain-e3-v2" { + return fmt.Errorf("unsupported linechain durable protocol %q", wire.DurableProtocol) + } + rawFields, err := decodeUniqueJSONObject(raw) + if err != nil { + return fmt.Errorf("decode linechain document fields: %w", err) + } + for _, name := range []string{"version", "durable_protocol", "operation", "fragment_basename", "fragment", "sidecar_patch", "previous_fragment_sha256", "fragment_sha256", "sidecar_patch_sha256", "artifact_sha256"} { + if _, ok := rawFields[name]; !ok { + return fmt.Errorf("linechain document field %s must be present", name) + } + } if filepath.Base(wire.FragmentBasename) != wire.FragmentBasename || !linechainBasenameRE.MatchString(wire.FragmentBasename) { return fmt.Errorf("fragment_basename is invalid") } + patch, patchCanonical, err := canonicalSemanticSidecarPatch(rawFields["sidecar_patch"]) + if err != nil { + return err + } + binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: wire.Operation, FragmentBasename: wire.FragmentBasename, + PreviousFragmentSHA256: wire.PreviousFragmentSHA256, FragmentSHA256: wire.FragmentSHA256, SidecarPatchSHA256: wire.SidecarPatchSHA256} + if err := verifySemanticArtifactBinding(wire.Fragment, patchCanonical, binding, wire.ArtifactSHA256); err != nil { + return err + } d := Document{ - Version: wire.Version, Operation: wire.Operation, ConfigDir: m.configDir, + Version: wire.Version, DurableProtocol: wire.DurableProtocol, Operation: wire.Operation, ConfigDir: m.configDir, FragmentBasename: wire.FragmentBasename, FragmentPath: filepath.Join(m.configDir, wire.FragmentBasename), SidecarPath: m.sidecarPath, - PreviousFragmentSHA256: wire.PreviousFragmentSHA256, Fragment: wire.Fragment, Sidecar: wire.Sidecar, - FragmentSHA256: wire.FragmentSHA256, SidecarSHA256: wire.SidecarSHA256, CombinedSHA256: wire.CombinedSHA256, + PreviousFragmentSHA256: wire.PreviousFragmentSHA256, Fragment: wire.Fragment, SidecarPatch: patch, + FragmentSHA256: wire.FragmentSHA256, SidecarPatchSHA256: wire.SidecarPatchSHA256, ArtifactSHA256: wire.ArtifactSHA256, + SidecarPatchCanonical: patchCanonical, } if err := m.validateDocument(d); err != nil { return err @@ -344,18 +377,26 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskS if err != nil { return err } - if err := requirePrevious("fragment", fragmentOld, fragmentHad, d.PreviousFragmentSHA256); err != nil { + previousFragmentSHA := "" + if d.PreviousFragmentSHA256 != nil { + previousFragmentSHA = *d.PreviousFragmentSHA256 + } + if err := requirePrevious("fragment", fragmentOld, fragmentHad, previousFragmentSHA); err != nil { return err } - mergedSidecar, err := mergeSidecar(sidecarOld, sidecarHad, d.Sidecar) + if !sidecarHad { + return fmt.Errorf("current semantic sidecar is required") + } + mergedSidecar, err := mergeManagedSidecar(sidecarOld, d.SidecarPatch) if err != nil { return err } - d.Sidecar = mergedSidecar - d = BindDocument(d) + mergedSidecarText := string(mergedSidecar) + d.SidecarOutput = &mergedSidecarText j := journal{Version: journalVersion, TaskID: taskID, LeaseID: leaseID, FragmentPath: d.FragmentPath, SidecarPath: d.SidecarPath, FragmentOld: digestMaybe(fragmentOld, fragmentHad), SidecarOld: digestMaybe(sidecarOld, sidecarHad), FragmentHadOld: fragmentHad, SidecarHadOld: sidecarHad, - FragmentNew: digestPtr(d.Fragment), SidecarNew: digestPtr(d.Sidecar), CombinedNew: d.CombinedSHA256, TaskScriptSHA: scriptBinding, Phase: "prepared"} + ArtifactSHA256: d.ArtifactSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256, + FragmentOutputSHA256: digestPtr(d.Fragment), SidecarOutputSHA256: digestPtr(d.SidecarOutput), TaskScriptSHA: scriptBinding, Phase: "prepared"} if err := m.writeBackup(path+".fragment.old", fragmentOld, fragmentHad); err != nil { return err } @@ -372,7 +413,7 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskS if err := m.writeJournal(path, j); err != nil { return err } - if err := m.publishFile(d.SidecarPath, d.Sidecar); err != nil { + if err := m.publishFile(d.SidecarPath, d.SidecarOutput); err != nil { return m.rollback(ctx, path, &j, d, fmt.Errorf("publish sidecar: %w", err)) } j.Phase = "pair_published" @@ -386,49 +427,38 @@ func (m *Manager) Apply(ctx context.Context, r io.Reader, taskID, leaseID, taskS return m.writeJournal(path, j) } -func mergeSidecar(current []byte, currentExists bool, desired *string) (*string, error) { - if desired == nil { - return nil, nil - } - merged := map[string]any{} - if currentExists { - if err := json.Unmarshal(current, &merged); err != nil { - return nil, fmt.Errorf("decode current semantic sidecar: %w", err) +func decodeUniqueJSONObject(raw []byte) (map[string]json.RawMessage, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + token, err := dec.Token() + if err != nil || token != json.Delim('{') { + return nil, fmt.Errorf("must be an object") + } + fields := make(map[string]json.RawMessage) + for dec.More() { + token, err := dec.Token() + if err != nil { + return nil, err + } + name, ok := token.(string) + if !ok { + return nil, fmt.Errorf("object field name is invalid") + } + if _, exists := fields[name]; exists { + return nil, fmt.Errorf("duplicate field %s", name) } - if err := validateSidecarObject("current", merged); err != nil { + var value json.RawMessage + if err := dec.Decode(&value); err != nil { return nil, err } + fields[name] = value } - want := map[string]any{} - if err := json.Unmarshal([]byte(*desired), &want); err != nil { - return nil, fmt.Errorf("decode desired semantic sidecar: %w", err) - } - if err := validateSidecarObject("desired", want); err != nil { + if _, err := dec.Token(); err != nil { return nil, err } - for key, value := range want { - merged[key] = value - } - b, err := json.Marshal(merged) - if err != nil { - return nil, fmt.Errorf("encode semantic sidecar: %w", err) + if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("unexpected trailing data") } - b = append(b, '\n') - s := string(b) - return &s, nil -} - -func validateSidecarObject(label string, value map[string]any) error { - if value == nil { - return fmt.Errorf("%s semantic sidecar must be a non-null object", label) - } - if value["schema"] != "lattice.singbox-metadata.v2" { - return fmt.Errorf("%s semantic sidecar has unsupported schema", label) - } - if _, ok := value["inbounds"].([]any); !ok { - return fmt.Errorf("%s semantic sidecar inbounds must be an array", label) - } - return nil + return fields, nil } // JournalRef is the bounded identity needed to cross-check outbox authority. @@ -438,12 +468,12 @@ type JournalRef struct { Result *model.TaskResult Terminal bool TaskScriptSHA string - CombinedSHA256 string + ArtifactSHA256 string JournalSHA256 string } type RecoveryAuthority struct { - TaskScriptSHA, CombinedSHA256, Phase, ResultSHA256, JournalSHA256 string + TaskScriptSHA, ArtifactSHA256, Phase, ResultSHA256, JournalSHA256 string } func (m *Manager) Snapshot() ([]JournalRef, error) { @@ -475,7 +505,7 @@ func (m *Manager) Snapshot() ([]JournalRef, error) { if err := m.validateJournal(filepath.Join(m.dir, entry.Name()), j); err != nil { return nil, err } - refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID, Phase: j.Phase, Result: j.Result, Terminal: journalTerminal(j.Phase), TaskScriptSHA: j.TaskScriptSHA, CombinedSHA256: j.CombinedNew, JournalSHA256: journalSHA(j)}) + refs = append(refs, JournalRef{TaskID: j.TaskID, LeaseID: j.LeaseID, Phase: j.Phase, Result: j.Result, Terminal: journalTerminal(j.Phase), TaskScriptSHA: j.TaskScriptSHA, ArtifactSHA256: j.ArtifactSHA256, JournalSHA256: journalSHA(j)}) } return refs, nil } @@ -510,7 +540,8 @@ func (m *Manager) validateJournal(path string, j journal) error { } if (j.FragmentHadOld && !validSHA(j.FragmentOld)) || (!j.FragmentHadOld && j.FragmentOld != "") || (j.SidecarHadOld && !validSHA(j.SidecarOld)) || (!j.SidecarHadOld && j.SidecarOld != "") || - (j.FragmentNew != "" && !validSHA(j.FragmentNew)) || !validSHA(j.SidecarNew) || !validSHA(j.CombinedNew) || !validSHA(j.TaskScriptSHA) { + (j.FragmentOutputSHA256 != "" && !validSHA(j.FragmentOutputSHA256)) || !validSHA(j.SidecarOutputSHA256) || + !validSHA(j.SidecarPatchSHA256) || !validSHA(j.ArtifactSHA256) || !validSHA(j.TaskScriptSHA) { return fmt.Errorf("linechain journal artifact digest shape is invalid") } if terminal { @@ -562,28 +593,32 @@ func (m *Manager) validateDocument(d Document) error { if d.Version != 2 { return fmt.Errorf("unsupported linechain document version %d", d.Version) } + if d.DurableProtocol != "linechain-e3-v2" { + return fmt.Errorf("unsupported linechain durable protocol %q", d.DurableProtocol) + } if d.FragmentBasename == "" { return fmt.Errorf("v2 fragment_basename is required") } switch d.Operation { case "create": - if d.PreviousFragmentSHA256 != "" || d.Fragment == nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 != nil || d.Fragment == nil || d.FragmentSHA256 == nil || d.SidecarPatch.DesiredDownstreamLineUUID == nil { return fmt.Errorf("create document has inconsistent old/desired shape") } case "replace": - if d.PreviousFragmentSHA256 == "" || d.Fragment == nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 == nil || d.Fragment == nil || d.FragmentSHA256 == nil || d.SidecarPatch.ExpectedDownstreamLineUUID == nil || d.SidecarPatch.DesiredDownstreamLineUUID == nil { return fmt.Errorf("replace document has inconsistent old/desired shape") } case "remove": - if d.PreviousFragmentSHA256 == "" || d.Fragment != nil || d.Sidecar == nil { + if d.PreviousFragmentSHA256 == nil || d.Fragment != nil || d.FragmentSHA256 != nil || d.SidecarPatch.ExpectedDownstreamLineUUID == nil || d.SidecarPatch.DesiredDownstreamLineUUID != nil { return fmt.Errorf("remove document has inconsistent old/desired shape") } default: return fmt.Errorf("unsupported linechain operation %q", d.Operation) } - bound := BindDocument(d) - if d.FragmentSHA256 != bound.FragmentSHA256 || d.SidecarSHA256 != bound.SidecarSHA256 || d.CombinedSHA256 != bound.CombinedSHA256 { - return fmt.Errorf("linechain desired artifact digest binding mismatch") + binding := semanticArtifactBindingV2{Schema: semanticArtifactSchema, Operation: d.Operation, FragmentBasename: d.FragmentBasename, + PreviousFragmentSHA256: d.PreviousFragmentSHA256, FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256} + if err := verifySemanticArtifactBinding(d.Fragment, d.SidecarPatchCanonical, binding, d.ArtifactSHA256); err != nil { + return err } configDir := filepath.Clean(d.ConfigDir) if !m.Configured() { @@ -620,8 +655,8 @@ func (m *Manager) validateDocument(d Document) error { if err := validateParents(d.SidecarPath); err != nil { return err } - for _, want := range []string{d.PreviousFragmentSHA256} { - if want != "" && !validSHA(want) { + for _, want := range []*string{d.PreviousFragmentSHA256, d.FragmentSHA256} { + if want != nil && !validSHA(*want) { return fmt.Errorf("previous artifact digest is invalid") } } @@ -750,7 +785,7 @@ func (m *Manager) recoverOne(ctx context.Context, path string, complete func(mod } if len(expected) > 0 && expected[0] != nil { want := expected[0] - if journalSHA(j) != want.JournalSHA256 || j.TaskScriptSHA != want.TaskScriptSHA || j.CombinedNew != want.CombinedSHA256 || j.Phase != want.Phase || resultSHA(j.Result) != want.ResultSHA256 { + if journalSHA(j) != want.JournalSHA256 || j.TaskScriptSHA != want.TaskScriptSHA || j.ArtifactSHA256 != want.ArtifactSHA256 || j.Phase != want.Phase || resultSHA(j.Result) != want.ResultSHA256 { return fmt.Errorf("linechain journal changed after authority capture") } } @@ -853,7 +888,7 @@ func pairMatches(j journal, desired bool) bool { return false } if desired { - return digestMaybe(a, ah) == j.FragmentNew && digestMaybe(b, bh) == j.SidecarNew + return digestMaybe(a, ah) == j.FragmentOutputSHA256 && digestMaybe(b, bh) == j.SidecarOutputSHA256 } return ah == j.FragmentHadOld && bh == j.SidecarHadOld && digestMaybe(a, ah) == j.FragmentOld && digestMaybe(b, bh) == j.SidecarOld } diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index 863a451..c14b789 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "io" "os" "os/exec" "path/filepath" @@ -27,9 +28,11 @@ func TestApplyCreateResolveAndCleanup(t *testing.T) { if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } + if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"credential": "sidecar-secret"})), 0o600); err != nil { + t.Fatal(err) + } fragment := `{"outbounds":[{"type":"direct","tag":"chain"}]}` - sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"credential":"sidecar-secret"}` - doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, Sidecar: &sidecar} + doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &fragment, SidecarPatch: testSidecarPatch("create")} applyDoc(t, m, doc, "task-a", "lease-a") result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: "task-a", LeaseID: "lease-a"}, model.TaskResult{TaskID: "task-a", LeaseID: "lease-a", ExitCode: 0, FinishedAt: time.Now().UTC()}) if err != nil || result.ExitCode != 0 { @@ -63,8 +66,7 @@ func TestApplyRejectsUnexpectedExistingAndSymlink(t *testing.T) { t.Fatal(err) } desired := "new" - desiredSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &desired, Sidecar: &desiredSidecar} + doc := Document{Version: 2, Operation: "create", FragmentBasename: filepath.Base(fragmentPath), Fragment: &desired, SidecarPatch: testSidecarPatch("create")} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "unexpected existing") { t.Fatalf("unexpected error: %v", err) } @@ -91,14 +93,14 @@ func TestCheckFailureRestoresExactOldPair(t *testing.T) { if err := m.ConfigureCommands("false", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - oldFragment, oldSidecar := "old-fragment", `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"old"}` + oldFragment, oldSidecar := "old-fragment", testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": "old"}) for path, data := range map[string]string{fragmentPath: oldFragment, sidecarPath: oldSidecar} { if err := os.WriteFile(path, []byte(data), 0o600); err != nil { t.Fatal(err) } } - newFragment, newSidecar := "new-fragment", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - doc := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + newFragment := "new-fragment" + doc := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")} if err := applyDocErr(m, doc, "task-a", "lease-a"); err == nil || !strings.Contains(err.Error(), "check failed") { t.Fatalf("unexpected error: %v", err) } @@ -119,8 +121,8 @@ func TestRecoveryProducesStableFailureAndCleansAfterCompletion(t *testing.T) { t.Fatal(err) } j := journal{ - Version: 1, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, - FragmentNew: digest([]byte("new-fragment")), SidecarNew: digest([]byte("new-sidecar")), CombinedNew: digest([]byte("combined")), TaskScriptSHA: digest(nil), Phase: "prepared", + Version: journalVersion, TaskID: "task-a", LeaseID: "lease-a", FragmentPath: fragmentPath, SidecarPath: sidecarPath, + FragmentOutputSHA256: digest([]byte("new-fragment")), SidecarOutputSHA256: digest([]byte("new-sidecar")), ArtifactSHA256: digest([]byte("combined")), SidecarPatchSHA256: digest([]byte("patch")), TaskScriptSHA: digest(nil), Phase: "prepared", } path := m.journalPath(j.TaskID, j.LeaseID) if err := writeJSON(path, j); err != nil { @@ -176,12 +178,11 @@ func TestValidateDocumentBindsOwnedPaths(t *testing.T) { configDir := filepath.Join(root, "conf") mustMkdir(t, configDir) fragment := "{}" - sidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` if err := m.ConfigureLayout(configDir, filepath.Join(root, "lattice-metadata.json")); err != nil { t.Fatal(err) } basename := "lattice-linechain-0123456789abcdef0123.json" - valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: m.configDir, FragmentBasename: basename, FragmentPath: filepath.Join(m.configDir, basename), SidecarPath: m.sidecarPath, Fragment: &fragment, Sidecar: &sidecar}) + valid := BindDocument(Document{Version: 2, Operation: "create", ConfigDir: m.configDir, FragmentBasename: basename, FragmentPath: filepath.Join(m.configDir, basename), SidecarPath: m.sidecarPath, Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}) if err := m.validateDocument(valid); err != nil { t.Fatal(err) } @@ -223,11 +224,14 @@ func TestApplyRejectsTrailingAndLegacyWireFields(t *testing.T) { if err := m.ConfigureLayout(configDir, filepath.Join(dir, "lattice-metadata.json")); err != nil { t.Fatal(err) } - fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := BindDocument(Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}) - base, _ := json.Marshal(wireDocumentV2{Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, Fragment: d.Fragment, Sidecar: d.Sidecar, FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256}) + fragment := `{}` + d := BindDocument(Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}) + base, _ := json.Marshal(wireDocumentV2{Version: d.Version, DurableProtocol: d.DurableProtocol, Operation: d.Operation, FragmentBasename: d.FragmentBasename, + Fragment: d.Fragment, SidecarPatch: d.SidecarPatch, PreviousFragmentSHA256: d.PreviousFragmentSHA256, + FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256, ArtifactSHA256: d.ArtifactSHA256}) for name, raw := range map[string]string{ "trailing": string(base) + ` {}`, + "duplicate known field": strings.Replace(string(base), `"version":2`, `"version":2,"version":2`, 1), "legacy config path": strings.TrimSuffix(string(base), "}") + `,"config_dir":""}`, "legacy fragment path": strings.TrimSuffix(string(base), "}") + `,"fragment_path":""}`, "legacy sidecar path": strings.TrimSuffix(string(base), "}") + `,"sidecar_path":""}`, @@ -265,7 +269,7 @@ func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { configDir := filepath.Join(dir, "conf") mustMkdir(t, configDir) sidecarPath := filepath.Join(dir, "lattice-metadata.json") - if err := os.WriteFile(sidecarPath, []byte(`{"schema":"lattice.singbox-metadata.v2","ordinary":{"owner":"sb"},"inbounds":[{"tag":"old"}]}`), 0o600); err != nil { + if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"ordinary": map[string]any{"owner": "sb"}})), 0o600); err != nil { t.Fatal(err) } if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { @@ -274,8 +278,8 @@ func TestSemanticSidecarOverlayPreservesOrdinaryFields(t *testing.T) { if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"managed"}]}` - d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar} + fragment := `{}` + d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")} applyDoc(t, m, d, "task-overlay", "lease-overlay") b, err := os.ReadFile(sidecarPath) if err != nil { @@ -352,8 +356,8 @@ func TestApplyRejectsSymlinkAndInvalidCurrentSidecar(t *testing.T) { if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { t.Fatal(err) } - fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar} + fragment := `{}` + d := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")} if err := applyDocErr(m, d, "task-"+name, "lease"); err == nil { t.Fatal("invalid current sidecar accepted") } @@ -438,17 +442,6 @@ func TestSnapshotRejectsSymlinkFIFOOwnerAndMode(t *testing.T) { } } -func TestBindDocumentMatchesCrossContractDigestVector(t *testing.T) { - fragment := "{\"outbounds\":[]}\n" - sidecar := "{\"inbounds\":[],\"schema\":\"lattice.singbox-metadata.v2\"}\n" - d := BindDocument(Document{Fragment: &fragment, Sidecar: &sidecar}) - if d.FragmentSHA256 != "0f4c04e3dff1fb9208c856b6de82f766bf6d99ff25ef30c68e076075a8d44b28" || - d.SidecarSHA256 != "f2eee32252fab6c99a728ebb1f16122798aff8609ace953f5f338d8f80b6b979" || - d.CombinedSHA256 != "155ec43b76241d49eac9d3a1415df33d84571f02363ac42bae04b0627e2ab848" { - t.Fatalf("cross-contract digest drift: %+v", d) - } -} - func TestRecoveryPhasesRestoreOrCommitExactPair(t *testing.T) { for _, tc := range []struct { phase string @@ -475,7 +468,7 @@ func TestRecoveryPhasesRestoreOrCommitExactPair(t *testing.T) { t.Fatal(err) } oldFragment := []byte("old-fragment") - oldSidecar := []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}`) + oldSidecar := []byte(testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true})) newFragment := []byte("new-fragment") newSidecar := []byte(`{"inbounds":[],"schema":"lattice.singbox-metadata.v2"}` + "\n") currentFragment, currentSidecar := newFragment, newSidecar @@ -492,8 +485,8 @@ func TestRecoveryPhasesRestoreOrCommitExactPair(t *testing.T) { } j := journal{ Version: journalVersion, TaskID: "task-" + tc.phase, LeaseID: "lease", FragmentPath: fragmentPath, SidecarPath: sidecarPath, - FragmentOld: digest(oldFragment), SidecarOld: digest(oldSidecar), FragmentNew: digest(newFragment), SidecarNew: digest(newSidecar), - CombinedNew: digest(append(append(append([]byte{}, newFragment...), 0), newSidecar...)), TaskScriptSHA: digest(nil), + FragmentOld: digest(oldFragment), SidecarOld: digest(oldSidecar), FragmentOutputSHA256: digest(newFragment), SidecarOutputSHA256: digest(newSidecar), + ArtifactSHA256: digest(append(append(append([]byte{}, newFragment...), 0), newSidecar...)), SidecarPatchSHA256: digest([]byte("patch")), TaskScriptSHA: digest(nil), FragmentHadOld: true, SidecarHadOld: true, Phase: tc.phase, } path := m.journalPath(j.TaskID, j.LeaseID) @@ -544,7 +537,7 @@ func TestJournalAuthorityRejectsCorruptSemanticsAndUnconfiguredRecovery(t *testi for name, mutate := range map[string]func(*journal){ "phase": func(j *journal) { j.Phase = "invented" }, "path": func(j *journal) { j.FragmentPath = filepath.Join(t.TempDir(), filepath.Base(j.FragmentPath)) }, - "digest": func(j *journal) { j.SidecarNew = "bad" }, + "digest": func(j *journal) { j.SidecarOutputSHA256 = "bad" }, "result": func(j *journal) { j.Phase = "terminal_desired" j.Result = &model.TaskResult{TaskID: "wrong", LeaseID: j.LeaseID, FinishedAt: time.Now().UTC()} @@ -589,13 +582,13 @@ func TestAuthorizedRecoveryRejectsJournalSwapAfterSnapshot(t *testing.T) { } ref := refs[0] authority := map[string]RecoveryAuthority{ - ref.TaskID + "\x00" + ref.LeaseID: {TaskScriptSHA: ref.TaskScriptSHA, CombinedSHA256: ref.CombinedSHA256, Phase: ref.Phase, ResultSHA256: resultSHA(ref.Result), JournalSHA256: ref.JournalSHA256}, + ref.TaskID + "\x00" + ref.LeaseID: {TaskScriptSHA: ref.TaskScriptSHA, ArtifactSHA256: ref.ArtifactSHA256, Phase: ref.Phase, ResultSHA256: resultSHA(ref.Result), JournalSHA256: ref.JournalSHA256}, } j, err := readJournal(path) if err != nil { t.Fatal(err) } - j.CombinedNew = digest([]byte("swapped-artifact")) + j.ArtifactSHA256 = digest([]byte("swapped-artifact")) if err := writeJSON(path, j); err != nil { t.Fatal(err) } @@ -674,7 +667,7 @@ func TestApplyExactRetryReplaceAndRemove(t *testing.T) { configDir := filepath.Join(root, "conf") mustMkdir(t, configDir) sidecarPath := filepath.Join(root, "meta.json") - if err := os.WriteFile(sidecarPath, []byte(`{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"keep"}`), 0o600); err != nil { + if err := os.WriteFile(sidecarPath, []byte(testCurrentSidecar(nil, map[string]any{"ordinary": "keep"})), 0o600); err != nil { t.Fatal(err) } if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { @@ -686,8 +679,8 @@ func TestApplyExactRetryReplaceAndRemove(t *testing.T) { runs := 0 m.run = func(context.Context, string, ...string) ([]byte, error) { runs++; return nil, nil } basename := "lattice-linechain-0123456789abcdef0123.json" - fragment1, sidecar1 := "one", `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"one"}]}` - create := Document{Version: 2, Operation: "create", FragmentBasename: basename, Fragment: &fragment1, Sidecar: &sidecar1} + fragment1 := "one" + create := Document{Version: 2, Operation: "create", FragmentBasename: basename, Fragment: &fragment1, SidecarPatch: testSidecarPatch("create")} applyDoc(t, m, create, "create", "lease") if err := m.Apply(context.Background(), bytes.NewReader(marshalDocument(t, create)), "create", "lease", digest([]byte("different-script"))); err == nil || !strings.Contains(err.Error(), "does not match") { t.Fatalf("mismatched retry binding error=%v", err) @@ -708,12 +701,11 @@ func TestApplyExactRetryReplaceAndRemove(t *testing.T) { t.Fatal(err) } - fragment2, sidecar2 := "two", `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"two"}]}` - replace := Document{Version: 2, Operation: "replace", FragmentBasename: basename, PreviousFragmentSHA256: digest([]byte(fragment1)), Fragment: &fragment2, Sidecar: &sidecar2} + fragment2 := "two" + replace := Document{Version: 2, Operation: "replace", FragmentBasename: basename, PreviousFragmentSHA256: stringPtr(digest([]byte(fragment1))), Fragment: &fragment2, SidecarPatch: testSidecarPatch("replace")} applyDoc(t, m, replace, "replace", "lease") resolveSuccess(t, m, "replace", "lease") - removeSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - remove := Document{Version: 2, Operation: "remove", FragmentBasename: basename, PreviousFragmentSHA256: digest([]byte(fragment2)), Sidecar: &removeSidecar} + remove := Document{Version: 2, Operation: "remove", FragmentBasename: basename, PreviousFragmentSHA256: stringPtr(digest([]byte(fragment2))), SidecarPatch: testSidecarPatch("remove")} applyDoc(t, m, remove, "remove", "lease") resolveSuccess(t, m, "remove", "lease") if _, err := os.Stat(filepath.Join(m.configDir, basename)); !errors.Is(err, os.ErrNotExist) { @@ -737,7 +729,7 @@ func TestCommandAndRollbackFailureMatrix(t *testing.T) { mustMkdir(t, configDir) sidecarPath := filepath.Join(root, "meta.json") oldFragment := "old" - oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":"secret-not-logged"}` + oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": "secret-not-logged"}) fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { t.Fatal(err) @@ -757,8 +749,8 @@ func TestCommandAndRollbackFailureMatrix(t *testing.T) { } return nil, nil } - newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + newFragment := "new" + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")} err := applyDocErr(m, d, "task-"+failAt, "lease") if err == nil || strings.Contains(err.Error(), "secret-not-logged") || strings.Contains(err.Error(), "private-key-canary") { t.Fatalf("failure=%v", err) @@ -803,7 +795,7 @@ func TestJournalAndPublishFailureMatrix(t *testing.T) { fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(root, "meta.json") oldFragment := "old" - oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}` + oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true}) if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { t.Fatal(err) } @@ -832,8 +824,8 @@ func TestJournalAndPublishFailureMatrix(t *testing.T) { } return writeJSON(path, value) }) - newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + newFragment := "new" + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")} err := applyDocErr(m, d, "task-"+failAt, "lease") if err == nil || !failed { t.Fatalf("injected failure not surfaced: %v", err) @@ -854,7 +846,7 @@ func TestPostRunAndRecoveryRuntimeVerificationFailures(t *testing.T) { fragmentPath := filepath.Join(configDir, "lattice-linechain-0123456789abcdef0123.json") sidecarPath := filepath.Join(root, "meta.json") oldFragment := "old" - oldSidecar := `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"ordinary":true}` + oldSidecar := testCurrentSidecar(stringPtr(newUUID), map[string]any{"ordinary": true}) if err := os.WriteFile(fragmentPath, []byte(oldFragment), 0o600); err != nil { t.Fatal(err) } @@ -868,8 +860,8 @@ func TestPostRunAndRecoveryRuntimeVerificationFailures(t *testing.T) { t.Fatal(err) } m.run = func(context.Context, string, ...string) ([]byte, error) { return nil, nil } - newFragment, newSidecar := "new", `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: digest([]byte(oldFragment)), Fragment: &newFragment, Sidecar: &newSidecar} + newFragment := "new" + d := Document{Version: 2, Operation: "replace", FragmentBasename: filepath.Base(fragmentPath), PreviousFragmentSHA256: stringPtr(digest([]byte(oldFragment))), Fragment: &newFragment, SidecarPatch: testSidecarPatch("replace")} applyDoc(t, m, d, "task-post-run", "lease") return m, fragmentPath, sidecarPath, oldSidecar } @@ -947,8 +939,8 @@ func appliedJournal(t *testing.T, taskID, leaseID string) (*Manager, string) { if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { t.Fatal(err) } - fragment, sidecar := `{}`, `{"schema":"lattice.singbox-metadata.v2","inbounds":[]}` - applyDoc(t, m, Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, Sidecar: &sidecar}, taskID, leaseID) + fragment := `{}` + applyDoc(t, m, Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")}, taskID, leaseID) return m, m.journalPath(taskID, leaseID) } @@ -975,22 +967,177 @@ func applyDoc(t *testing.T, m *Manager, d Document, task, lease string) { } } func applyDocErr(m *Manager, d Document, task, lease string) error { + d = BindDocument(d) + if m.sidecarPath != "" { + if _, err := os.Lstat(m.sidecarPath); errors.Is(err, os.ErrNotExist) { + inbound := map[string]any{"tag": d.SidecarPatch.SourceInboundTag, "line_uuid": d.SidecarPatch.SourceLineUUID} + if d.SidecarPatch.ExpectedDownstreamLineUUID != nil { + inbound["chain"] = map[string]any{"downstream_line_uuid": *d.SidecarPatch.ExpectedDownstreamLineUUID} + } + base, _ := json.Marshal(map[string]any{"schema": semanticSidecarMetadataSchema, "inbounds": []any{inbound}}) + if err := os.WriteFile(m.sidecarPath, base, 0o600); err != nil { + return err + } + } + } b := marshalDocument(nil, d) return m.Apply(context.Background(), strings.NewReader(string(b)), task, lease, digest([]byte(task))) } +func testSidecarPatch(operation string) SidecarPatchV2 { + patch := SidecarPatchV2{Schema: semanticSidecarPatchSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", DesiredDownstreamLineUUID: stringPtr(newUUID)} + if operation != "create" { + patch.ExpectedDownstreamLineUUID = stringPtr(newUUID) + } + if operation == "remove" { + patch.DesiredDownstreamLineUUID = nil + } + return patch +} + func marshalDocument(t *testing.T, d Document) []byte { d = BindDocument(d) b, err := json.Marshal(wireDocumentV2{ - Version: d.Version, Operation: d.Operation, FragmentBasename: d.FragmentBasename, - PreviousFragmentSHA256: d.PreviousFragmentSHA256, Fragment: d.Fragment, Sidecar: d.Sidecar, - FragmentSHA256: d.FragmentSHA256, SidecarSHA256: d.SidecarSHA256, CombinedSHA256: d.CombinedSHA256, + Version: d.Version, DurableProtocol: d.DurableProtocol, Operation: d.Operation, FragmentBasename: d.FragmentBasename, + Fragment: d.Fragment, SidecarPatch: d.SidecarPatch, PreviousFragmentSHA256: d.PreviousFragmentSHA256, + FragmentSHA256: d.FragmentSHA256, SidecarPatchSHA256: d.SidecarPatchSHA256, ArtifactSHA256: d.ArtifactSHA256, }) if err != nil && t != nil { t.Fatal(err) } return b } + +func TestConsumeProductionServerFixture(t *testing.T) { + fixturePath := os.Getenv("LATTICE_LINECHAIN_SERVER_FIXTURE") + if fixturePath == "" { + t.Skip("LATTICE_LINECHAIN_SERVER_FIXTURE is not set") + } + type fixture struct { + Schema string `json:"schema"` + ApprovalArtifactSHA256 string `json:"approval_artifact_sha256"` + RequestSHA256 string `json:"request_sha256"` + TaskScriptSHA256 string `json:"task_script_sha256"` + TaskID string `json:"task_id"` + LeaseID string `json:"lease_id"` + Document json.RawMessage `json:"document"` + } + rawFixture, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatal(err) + } + var f fixture + dec := json.NewDecoder(bytes.NewReader(rawFixture)) + dec.DisallowUnknownFields() + if err := dec.Decode(&f); err != nil { + t.Fatal(err) + } + if err := dec.Decode(new(any)); !errors.Is(err, io.EOF) { + t.Fatalf("fixture has trailing data: %v", err) + } + if f.Schema != "lattice.linechain.cross-contract-fixture.v2" || !validSHA(f.ApprovalArtifactSHA256) || !validSHA(f.RequestSHA256) || !validSHA(f.TaskScriptSHA256) || f.TaskID == "" || f.LeaseID == "" { + t.Fatalf("invalid production fixture wrapper: %+v", f) + } + var document wireDocumentV2 + if err := json.Unmarshal(f.Document, &document); err != nil { + t.Fatal(err) + } + if document.ArtifactSHA256 != f.ApprovalArtifactSHA256 { + t.Fatalf("approval/document artifact mismatch: %s != %s", f.ApprovalArtifactSHA256, document.ArtifactSHA256) + } + + root := t.TempDir() + m, err := Open(filepath.Join(root, "txn")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = m.Close() }) + configDir := filepath.Join(root, "conf") + if err := os.Mkdir(configDir, 0o700); err != nil { + t.Fatal(err) + } + sidecarPath := filepath.Join(root, "lattice-metadata.json") + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + current := []byte("{\n \"unknown_root\": {\"preserve\": true},\n \"schema\": \"lattice.singbox-metadata.v2\",\n \"inbounds\": [\n" + + " {\"tag\":\"before\",\"line_uuid\":\"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa\",\"keep\":1},\n" + + " {\"tag\":\"source-b\",\"line_uuid\":\"22222222-2222-4222-8222-222222222222\",\"ordinary\":\"keep\"},\n" + + " {\"tag\":\"after\",\"line_uuid\":\"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb\",\"keep\":2}\n ]\n}\n") + if err := os.WriteFile(sidecarPath, current, 0o600); err != nil { + t.Fatal(err) + } + if err := m.Apply(context.Background(), bytes.NewReader(f.Document), f.TaskID, f.LeaseID, f.TaskScriptSHA256); err != nil { + t.Fatal(err) + } + + j, err := readJournal(m.journalPath(f.TaskID, f.LeaseID)) + if err != nil { + t.Fatal(err) + } + fragmentBytes, err := os.ReadFile(filepath.Join(configDir, document.FragmentBasename)) + if err != nil { + t.Fatal(err) + } + sidecarBytes, err := os.ReadFile(sidecarPath) + if err != nil { + t.Fatal(err) + } + if j.ArtifactSHA256 != f.ApprovalArtifactSHA256 || j.TaskScriptSHA != f.TaskScriptSHA256 || j.FragmentOutputSHA256 != digest(fragmentBytes) || j.SidecarOutputSHA256 != digest(sidecarBytes) { + t.Fatalf("fixture journal authority/output mismatch: %+v", j) + } + var output struct { + UnknownRoot map[string]bool `json:"unknown_root"` + Inbounds []struct { + Tag string `json:"tag"` + Ordinary string `json:"ordinary"` + Keep int `json:"keep"` + Chain map[string]any `json:"chain"` + } `json:"inbounds"` + } + if err := json.Unmarshal(sidecarBytes, &output); err != nil { + t.Fatal(err) + } + wantDownstream := "" + if document.SidecarPatch.DesiredDownstreamLineUUID != nil { + wantDownstream = *document.SidecarPatch.DesiredDownstreamLineUUID + } + if !output.UnknownRoot["preserve"] || len(output.Inbounds) != 3 || output.Inbounds[0].Tag != "before" || output.Inbounds[0].Keep != 1 || output.Inbounds[1].Tag != "source-b" || output.Inbounds[1].Ordinary != "keep" || output.Inbounds[2].Tag != "after" || output.Inbounds[2].Keep != 2 || output.Inbounds[1].Chain["downstream_line_uuid"] != wantDownstream { + t.Fatalf("fixture merge did not preserve scoped host state: %s", sidecarBytes) + } + + m.publishFile = func(string, *string) error { return errors.New("replay attempted to republish output") } + if err := m.Apply(context.Background(), bytes.NewReader(f.Document), f.TaskID, f.LeaseID, f.TaskScriptSHA256); err != nil { + t.Fatal(err) + } + replayed, err := readJournal(m.journalPath(f.TaskID, f.LeaseID)) + if err != nil { + t.Fatal(err) + } + if replayed.Phase != "terminal_desired" || replayed.FragmentOutputSHA256 != j.FragmentOutputSHA256 || replayed.SidecarOutputSHA256 != j.SidecarOutputSHA256 { + t.Fatalf("fixture replay did not use journaled output authority: %+v", replayed) + } +} + +func testCurrentSidecar(expected *string, extras map[string]any) string { + inbound := map[string]any{"tag": "source", "line_uuid": sourceUUID} + if expected != nil { + inbound["chain"] = map[string]any{"downstream_line_uuid": *expected} + } + top := map[string]any{"schema": semanticSidecarMetadataSchema, "inbounds": []any{inbound}} + for key, value := range extras { + top[key] = value + } + raw, err := json.Marshal(top) + if err != nil { + panic(err) + } + return string(raw) +} + func assertFile(t *testing.T, p, want string) { t.Helper() b, err := os.ReadFile(p) diff --git a/internal/linechain/sidecar_patch.go b/internal/linechain/sidecar_patch.go index 4d2560e..dd98223 100644 --- a/internal/linechain/sidecar_patch.go +++ b/internal/linechain/sidecar_patch.go @@ -8,11 +8,15 @@ import ( "strings" ) -const semanticSidecarSchema = "lattice.singbox-metadata.v2" +const ( + semanticSidecarMetadataSchema = "lattice.singbox-metadata.v2" + semanticSidecarPatchSchema = "lattice.singbox-linechain-sidecar-patch.v1" + semanticArtifactSchema = "lattice.singbox-linechain-artifact.v2" +) var lowercaseUUIDv4RE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) -type semanticSidecarPatchV2 struct { +type SidecarPatchV2 struct { Schema string `json:"schema"` SourceLineUUID string `json:"source_line_uuid"` SourceInboundTag string `json:"source_inbound_tag"` @@ -20,72 +24,102 @@ type semanticSidecarPatchV2 struct { DesiredDownstreamLineUUID *string `json:"desired_downstream_line_uuid"` } -type semanticSidecarBinding struct { - PatchSHA256 string - ArtifactSHA256 string +type semanticArtifactBindingV2 struct { + Schema string `json:"schema"` + Operation string `json:"operation"` + FragmentBasename string `json:"fragment_basename"` + PreviousFragmentSHA256 *string `json:"previous_fragment_sha256"` + FragmentSHA256 *string `json:"fragment_sha256"` + SidecarPatchSHA256 string `json:"sidecar_patch_sha256"` } -func canonicalSemanticSidecarPatch(raw []byte) (semanticSidecarPatchV2, []byte, error) { +func canonicalSemanticSidecarPatch(raw []byte) (SidecarPatchV2, []byte, error) { var fields map[string]json.RawMessage if err := decodeJSONObject(raw, &fields); err != nil { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) + return SidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) } for _, name := range []string{"schema", "source_line_uuid", "source_inbound_tag", "expected_downstream_line_uuid", "desired_downstream_line_uuid"} { if _, ok := fields[name]; !ok { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch field %s must be present", name) + return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch field %s must be present", name) } } if len(fields) != 5 { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch has unknown fields") + return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch has unknown fields") } - var patch semanticSidecarPatchV2 + var patch SidecarPatchV2 if err := json.Unmarshal(raw, &patch); err != nil { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) + return SidecarPatchV2{}, nil, fmt.Errorf("decode semantic sidecar patch: %w", err) } - if patch.Schema != semanticSidecarSchema || !lowercaseUUIDv4RE.MatchString(patch.SourceLineUUID) || strings.TrimSpace(patch.SourceInboundTag) == "" || patch.SourceInboundTag != strings.TrimSpace(patch.SourceInboundTag) { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch identity is invalid") + if patch.Schema != semanticSidecarPatchSchema || !lowercaseUUIDv4RE.MatchString(patch.SourceLineUUID) || strings.TrimSpace(patch.SourceInboundTag) == "" || patch.SourceInboundTag != strings.TrimSpace(patch.SourceInboundTag) { + return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch identity is invalid") } for _, value := range []*string{patch.ExpectedDownstreamLineUUID, patch.DesiredDownstreamLineUUID} { if value != nil && !lowercaseUUIDv4RE.MatchString(*value) { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar downstream identity is invalid") + return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar downstream identity is invalid") } } canonical, err := json.Marshal(patch) if err != nil { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("encode semantic sidecar patch: %w", err) + return SidecarPatchV2{}, nil, fmt.Errorf("encode semantic sidecar patch: %w", err) } if !bytes.Equal(raw, canonical) { - return semanticSidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch is not canonical") + return SidecarPatchV2{}, nil, fmt.Errorf("semantic sidecar patch is not canonical") } return patch, canonical, nil } -func semanticArtifactDigest(fragment *string, patch []byte) string { - var raw []byte - if fragment != nil { - raw = append(raw, []byte(*fragment)...) - } - raw = append(raw, 0) - raw = append(raw, patch...) - return digest(raw) +func canonicalSemanticArtifactBinding(binding semanticArtifactBindingV2) ([]byte, error) { + if binding.Schema != semanticArtifactSchema || !linechainBasenameRE.MatchString(binding.FragmentBasename) || !validSHA(binding.SidecarPatchSHA256) { + return nil, fmt.Errorf("semantic artifact binding identity is invalid") + } + switch binding.Operation { + case "create": + if binding.PreviousFragmentSHA256 != nil || binding.FragmentSHA256 == nil { + return nil, fmt.Errorf("create semantic artifact binding has invalid fragment shape") + } + case "replace": + if binding.PreviousFragmentSHA256 == nil || binding.FragmentSHA256 == nil { + return nil, fmt.Errorf("replace semantic artifact binding has invalid fragment shape") + } + case "remove": + if binding.PreviousFragmentSHA256 == nil || binding.FragmentSHA256 != nil { + return nil, fmt.Errorf("remove semantic artifact binding has invalid fragment shape") + } + default: + return nil, fmt.Errorf("unsupported semantic artifact operation %q", binding.Operation) + } + for _, value := range []*string{binding.PreviousFragmentSHA256, binding.FragmentSHA256} { + if value != nil && !validSHA(*value) { + return nil, fmt.Errorf("semantic artifact fragment digest is invalid") + } + } + return json.Marshal(binding) } -func verifySemanticSidecarBinding(fragment *string, patch []byte, issued semanticSidecarBinding) error { - checks := []struct{ label, want, got string }{ - {label: "patch", want: issued.PatchSHA256, got: digest(patch)}, - {label: "artifact", want: issued.ArtifactSHA256, got: semanticArtifactDigest(fragment, patch)}, - } - for _, check := range checks { - if !validSHA(check.want) || check.want != check.got { - return fmt.Errorf("semantic sidecar %s digest binding mismatch", check.label) +func verifySemanticArtifactBinding(fragment *string, patch []byte, binding semanticArtifactBindingV2, issuedArtifactSHA256 string) error { + if binding.FragmentSHA256 == nil { + if fragment != nil { + return fmt.Errorf("semantic artifact fragment output must be absent") } + } else if fragment == nil || digest([]byte(*fragment)) != *binding.FragmentSHA256 { + return fmt.Errorf("semantic artifact fragment digest binding mismatch") + } + if digest(patch) != binding.SidecarPatchSHA256 { + return fmt.Errorf("semantic artifact sidecar patch digest binding mismatch") + } + canonical, err := canonicalSemanticArtifactBinding(binding) + if err != nil { + return err + } + if !validSHA(issuedArtifactSHA256) || digest(canonical) != issuedArtifactSHA256 { + return fmt.Errorf("semantic artifact digest binding mismatch") } return nil } // mergeManagedSidecar applies one authenticated source patch to an existing // metadata v2 sidecar. It changes only the matched inbound's chain object. -func mergeManagedSidecar(current []byte, patch semanticSidecarPatchV2) ([]byte, error) { +func mergeManagedSidecar(current []byte, patch SidecarPatchV2) ([]byte, error) { var top map[string]json.RawMessage if err := decodeJSONObject(current, &top); err != nil { return nil, fmt.Errorf("decode current semantic sidecar: %w", err) @@ -194,7 +228,7 @@ func nullableStringEqual(a, b *string) bool { func validateSemanticSidecarTop(top map[string]json.RawMessage) error { var schema string - if err := json.Unmarshal(top["schema"], &schema); err != nil || schema != semanticSidecarSchema { + if err := json.Unmarshal(top["schema"], &schema); err != nil || schema != semanticSidecarMetadataSchema { return fmt.Errorf("unsupported schema") } var inbounds []json.RawMessage diff --git a/internal/linechain/sidecar_patch_test.go b/internal/linechain/sidecar_patch_test.go index 6cf48a6..69e90f8 100644 --- a/internal/linechain/sidecar_patch_test.go +++ b/internal/linechain/sidecar_patch_test.go @@ -16,7 +16,7 @@ const ( func stringPtr(value string) *string { return &value } func patchBytes(expected, desired *string) []byte { - b, err := json.Marshal(semanticSidecarPatchV2{Schema: semanticSidecarSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", ExpectedDownstreamLineUUID: expected, DesiredDownstreamLineUUID: desired}) + b, err := json.Marshal(SidecarPatchV2{Schema: semanticSidecarPatchSchema, SourceLineUUID: sourceUUID, SourceInboundTag: "source", ExpectedDownstreamLineUUID: expected, DesiredDownstreamLineUUID: desired}) if err != nil { panic(err) } @@ -94,27 +94,79 @@ func TestMergeManagedSidecarRejectsIdentityAndBaseDrift(t *testing.T) { } } -func TestSemanticBindingNeverRewritesIssuedDigests(t *testing.T) { +func TestSemanticArtifactBindingNeverRewritesIssuedDigests(t *testing.T) { fragment := `{"outbounds":[]}` patch := patchBytes(nil, stringPtr(newUUID)) - issued := semanticSidecarBinding{PatchSHA256: digest(patch), ArtifactSHA256: semanticArtifactDigest(&fragment, patch)} - if err := verifySemanticSidecarBinding(&fragment, patch, issued); err != nil { + fragmentSHA := digest([]byte(fragment)) + binding := semanticArtifactBindingV2{ + Schema: semanticArtifactSchema, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", + PreviousFragmentSHA256: nil, FragmentSHA256: &fragmentSHA, SidecarPatchSHA256: digest(patch), + } + canonical, err := canonicalSemanticArtifactBinding(binding) + if err != nil { t.Fatal(err) } - for name, mutate := range map[string]func(*semanticSidecarBinding){ - "patch": func(v *semanticSidecarBinding) { v.PatchSHA256 = digest([]byte("other")) }, - "artifact": func(v *semanticSidecarBinding) { v.ArtifactSHA256 = digest([]byte("other")) }, + issuedArtifactSHA := digest(canonical) + if err := verifySemanticArtifactBinding(&fragment, patch, binding, issuedArtifactSHA); err != nil { + t.Fatal(err) + } + t.Run("patch", func(t *testing.T) { + mismatch := binding + mismatch.SidecarPatchSHA256 = digest([]byte("other")) + before := mismatch + if err := verifySemanticArtifactBinding(&fragment, patch, mismatch, issuedArtifactSHA); err == nil { + t.Fatal("patch mismatch accepted") + } + if mismatch != before { + t.Fatalf("issued binding rewritten: %+v -> %+v", before, mismatch) + } + }) + for name, value := range map[string]string{ + "fragment": digest([]byte("other")), + "artifact": digest([]byte("other artifact")), } { t.Run(name, func(t *testing.T) { - mismatch := issued - mutate(&mismatch) + mismatch := binding + artifactSHA := issuedArtifactSHA + if name == "fragment" { + mismatch.FragmentSHA256 = &value + } else { + artifactSHA = value + } before := mismatch - if err := verifySemanticSidecarBinding(&fragment, patch, mismatch); err == nil { + if err := verifySemanticArtifactBinding(&fragment, patch, mismatch, artifactSHA); err == nil { t.Fatal("mismatch accepted") } if mismatch != before { - t.Fatalf("issued digest rewritten: %+v -> %+v", before, mismatch) + t.Fatalf("issued binding rewritten: %+v -> %+v", before, mismatch) } }) } } + +func TestCanonicalSemanticBindingMatchesServerVector(t *testing.T) { + const ( + patchJSON = `{"schema":"lattice.singbox-linechain-sidecar-patch.v1","source_line_uuid":"22222222-2222-4222-8222-222222222222","source_inbound_tag":"source-b","expected_downstream_line_uuid":null,"desired_downstream_line_uuid":"11111111-1111-4111-8111-111111111111"}` + patchSHA = "7394c9367aa36d0e37e1e6bb70d3de70afc1d6792f56754741ba118ca2137188" + artifactJSON = `{"schema":"lattice.singbox-linechain-artifact.v2","operation":"create","fragment_basename":"lattice-linechain-0123456789abcdef0123.json","previous_fragment_sha256":null,"fragment_sha256":"0000000000000000000000000000000000000000000000000000000000000000","sidecar_patch_sha256":"7394c9367aa36d0e37e1e6bb70d3de70afc1d6792f56754741ba118ca2137188"}` + artifactSHA = "bb59094488756276a385921951eaac3e36dc604eb4a03c4cb2e1a52797aee261" + ) + patch, canonicalPatch, err := canonicalSemanticSidecarPatch([]byte(patchJSON)) + if err != nil { + t.Fatal(err) + } + if string(canonicalPatch) != patchJSON || digest(canonicalPatch) != patchSHA { + t.Fatalf("server patch vector drift: bytes=%s sha=%s", canonicalPatch, digest(canonicalPatch)) + } + zeroSHA := strings.Repeat("0", 64) + canonicalArtifact, err := canonicalSemanticArtifactBinding(semanticArtifactBindingV2{ + Schema: semanticArtifactSchema, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", + FragmentSHA256: &zeroSHA, SidecarPatchSHA256: digest(canonicalPatch), + }) + if err != nil { + t.Fatal(err) + } + if patch.SourceLineUUID != "22222222-2222-4222-8222-222222222222" || string(canonicalArtifact) != artifactJSON || digest(canonicalArtifact) != artifactSHA { + t.Fatalf("server artifact vector drift: bytes=%s sha=%s", canonicalArtifact, digest(canonicalArtifact)) + } +} diff --git a/internal/taskoutbox/outbox.go b/internal/taskoutbox/outbox.go index 443f3bf..7ba7ec9 100644 --- a/internal/taskoutbox/outbox.go +++ b/internal/taskoutbox/outbox.go @@ -250,7 +250,7 @@ func (s *Store) RecoverInterrupted(nodeID string) error { if entry.State != stateLeased { continue } - if entry.DurableProtocol == "linechain-e3-v1" { + if entry.DurableProtocol == "linechain-e3-v2" { return fmt.Errorf("leased E3 task %s requires linechain journal recovery before generic outbox recovery", entry.Task.ID) } now := time.Now().UTC() @@ -390,6 +390,9 @@ func (s *Store) read(key string) (Entry, error) { if entry.Version != entryVersion || entry.Task.ID == "" || entry.Task.LeaseID == "" { return Entry{}, fmt.Errorf("invalid task result journal: %s", path) } + if strings.HasPrefix(entry.DurableProtocol, "linechain-e3-") && entry.DurableProtocol != "linechain-e3-v2" { + return Entry{}, fmt.Errorf("unsupported persisted linechain durable protocol %q", entry.DurableProtocol) + } if key != entryKey(entry.Task.ID, entry.Task.LeaseID) { return Entry{}, fmt.Errorf("task result journal key mismatch: %s", path) } diff --git a/internal/taskoutbox/outbox_test.go b/internal/taskoutbox/outbox_test.go index 8607d7e..cee122b 100644 --- a/internal/taskoutbox/outbox_test.go +++ b/internal/taskoutbox/outbox_test.go @@ -395,6 +395,24 @@ func TestOpenRefusesConcurrentOwnerWithoutChangingJournal(t *testing.T) { defer second.Close() } +func TestOpenRejectsPersistedLegacyLinechainProtocol(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir) + if err != nil { + t.Fatal(err) + } + task := testTask() + if _, err := store.BeginWithProtocol(task, "linechain-e3-v1"); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + if _, err := Open(dir); err == nil || !strings.Contains(err.Error(), "unsupported persisted linechain durable protocol") { + t.Fatalf("legacy linechain outbox error = %v", err) + } +} + func testTask() model.Task { return model.Task{ ID: "task-a", LeaseID: "lease-a", Interpreter: "sh", Script: "echo done", From ad3dfcf11f257320ab3a911e2856f530db73e705 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 09:34:55 -0700 Subject: [PATCH 27/43] Reject ambiguous host sidecar authority before mutation Route every host sidecar root, inbound, and chain object through the existing token-based unique-key decoder. Duplicate authority fields now fail before fragment publication, sidecar replacement, or journal creation, while preserving the strict semantic merge contract established by E3 v2. Constraint: Host sidecar bytes are independently mutable and cannot use last-key-wins JSON semantics for authority-bearing fields Rejected: Validate only the matched inbound after json.Unmarshal | duplicate keys are already collapsed and their ambiguity is no longer observable Confidence: high Scope-risk: narrow Reversibility: clean Directive: All authority-bearing JSON objects must reject duplicate keys before semantic validation or mutation Tested: five Apply-level no-mutation duplicate cases; production server fixture; go test ./...; go test -race -cover ./...; go vet ./...; installer, release-workflow, proto, install/version/compat, secret, and diff gates Not-tested: Full real sing-box lifecycle E2E remains owned by Task 18 --- internal/linechain/manager_test.go | 50 +++++++++++++++++++++++++++++ internal/linechain/sidecar_patch.go | 7 ++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/internal/linechain/manager_test.go b/internal/linechain/manager_test.go index c14b789..3a2aeb5 100644 --- a/internal/linechain/manager_test.go +++ b/internal/linechain/manager_test.go @@ -365,6 +365,56 @@ func TestApplyRejectsSymlinkAndInvalidCurrentSidecar(t *testing.T) { } } +func TestApplyRejectsDuplicateSidecarAuthorityWithoutMutation(t *testing.T) { + const ( + uuid = `11111111-1111-4111-8111-1111111111aa` + downstream = `33333333-3333-4333-8333-333333333333` + ) + cases := map[string]string{ + "root inbounds": `{"schema":"lattice.singbox-metadata.v2","inbounds":[],"inbounds":[{"tag":"source","line_uuid":"` + uuid + `"}]}`, + "inbound line_uuid": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","line_uuid":"` + uuid + `"}]}`, + "inbound tag": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","tag":"source","line_uuid":"` + uuid + `"}]}`, + "inbound chain": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","chain":{"downstream_line_uuid":"` + downstream + `"},"chain":{"downstream_line_uuid":"` + downstream + `"}}]}`, + "chain downstream_line_uuid": `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source","line_uuid":"` + uuid + `","chain":{"downstream_line_uuid":"` + downstream + `","downstream_line_uuid":"` + downstream + `"}}]}`, + } + for name, current := range cases { + t.Run(name, func(t *testing.T) { + m, _ := testManager(t) + root := t.TempDir() + configDir := filepath.Join(root, "conf") + mustMkdir(t, configDir) + sidecarPath := filepath.Join(root, "lattice-metadata.json") + if err := os.WriteFile(sidecarPath, []byte(current), 0o600); err != nil { + t.Fatal(err) + } + if err := m.ConfigureLayout(configDir, sidecarPath); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("true", []string{"true"}, []string{"true"}); err != nil { + t.Fatal(err) + } + fragment := `{}` + doc := Document{Version: 2, Operation: "create", FragmentBasename: "lattice-linechain-0123456789abcdef0123.json", Fragment: &fragment, SidecarPatch: testSidecarPatch("create")} + if err := applyDocErr(m, doc, "task-duplicate-"+name, "lease"); err == nil || !strings.Contains(err.Error(), "duplicate field") { + t.Fatalf("duplicate sidecar authority error = %v", err) + } + assertFile(t, sidecarPath, current) + if _, err := os.Lstat(filepath.Join(configDir, doc.FragmentBasename)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fragment mutated before duplicate rejection: %v", err) + } + entries, err := os.ReadDir(m.dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".json") || strings.Contains(entry.Name(), ".old") { + t.Fatalf("journal mutation before duplicate rejection: %s", entry.Name()) + } + } + }) + } +} + func TestSnapshotRejectsRenamedAndDuplicateJournalIdentity(t *testing.T) { for _, duplicate := range []bool{false, true} { t.Run(map[bool]string{false: "renamed", true: "duplicate"}[duplicate], func(t *testing.T) { diff --git a/internal/linechain/sidecar_patch.go b/internal/linechain/sidecar_patch.go index dd98223..e8bc9af 100644 --- a/internal/linechain/sidecar_patch.go +++ b/internal/linechain/sidecar_patch.go @@ -257,11 +257,10 @@ func decodeJSONObject(raw []byte, dst *map[string]json.RawMessage) error { if len(bytes.TrimSpace(raw)) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { return fmt.Errorf("must be a non-null object") } - if err := json.Unmarshal(raw, dst); err != nil { + fields, err := decodeUniqueJSONObject(raw) + if err != nil { return err } - if *dst == nil { - return fmt.Errorf("must be a non-null object") - } + *dst = fields return nil } From cc2da567cfe61917f26055deee4647cf76f00ca0 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 10:19:35 -0700 Subject: [PATCH 28/43] Make the authentic linechain gate exercise the server lifecycle The mandatory script now builds the exact agent, locates the frozen repaired server tree, runs its persistent lifecycle gate, and then proves live A/B/client/origin traffic and recovery with the same official sing-box input. The agent fixture now uses the strict semantic v2 patch and artifact contract. Constraint: The public command and mandatory official sing-box 1.13.x input remain unchanged Rejected: Sequential fixture-only coverage | it would not exercise public HTTP planning, leasing, and result replay Confidence: high Scope-risk: narrow Reversibility: clean Directive: Do not add a skip path or replace official sing-box with a simulator Tested: Exact scripts/test-linechain-e2e.sh command with official sing-box 1.13.18; agent race compile; tagged vet Not-tested: GitHub CI --- cmd/lattice-agent/linechain_e2e_test.go | 70 +++++++++++++++---------- scripts/test-linechain-e2e.sh | 22 ++++++++ 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index 0c2fa74..8e2dfe5 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -126,9 +126,10 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { "route":{"rules":[{"inbound":["source-b"],"outbound":"chain-to-a"}]} } `, observer.Port(), uuidA, realityPublic) - sidecar := fmt.Sprintf(`{"inbounds":[{"tag":"source-b","line_uuid":%q,"chain":{"downstream_line_uuid":%q}}],"schema":"lattice.singbox-metadata.v2"} -`, lineUUIDB, lineUUIDA) - sidecar = canonicalJSONString(t, sidecar) + initialSidecar := canonicalJSONString(t, fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","unknown_root":{"keep":true},"inbounds":[{"tag":"unrelated-before","line_uuid":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","keep":1},{"tag":"source-b","line_uuid":%q,"ordinary":"keep"},{"tag":"unrelated-after","line_uuid":"dddddddd-dddd-4ddd-8ddd-dddddddddddd","keep":2}]}`, lineUUIDB)) + desiredSidecar := canonicalJSONString(t, fmt.Sprintf(`{"schema":"lattice.singbox-metadata.v2","unknown_root":{"keep":true},"inbounds":[{"tag":"unrelated-before","line_uuid":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","keep":1},{"tag":"source-b","line_uuid":%q,"ordinary":"keep","chain":{"downstream_line_uuid":%q}},{"tag":"unrelated-after","line_uuid":"dddddddd-dddd-4ddd-8ddd-dddddddddddd","keep":2}]}`, lineUUIDB, lineUUIDA)) + writeFile(t, sidecarPath, initialSidecar) + targetLineUUID := lineUUIDA m := openE2EManager(t, bin, root, bDir, sidecarPath, txnDir, bPort) defer m.Close() @@ -136,7 +137,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { // Crash a real apply helper process group after it publishes both artifacts, // while this test (the agent/supervisor analogue) remains alive. Recovery // must restore and restart B before any inventory, traffic, or result callback. - crashDoc := bindE2EDocument("create", basename, "", &fragment, &sidecar) + crashDoc := bindE2EDocument("create", basename, "", &fragment, lineUUIDB, "source-b", nil, &targetLineUUID) crashBytes := marshalE2EDocument(t, crashDoc) apply := exec.Command(os.Args[0], "-test.run=^TestLinechainE2EApplyHelper$", "--", root) crashMarker := filepath.Join(root, "restart-blocked") @@ -174,7 +175,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { } t.Cleanup(stopApply) waitForFileOrProcess(t, crashMarker, applyDone, &applyLog) - waitForExactPair(t, fragmentPath, fragment, sidecarPath, sidecar) + waitForExactPair(t, fragmentPath, fragment, sidecarPath, desiredSidecar) stopApply() order := []string{} @@ -216,9 +217,7 @@ func TestLinechainRealSingBoxE2E(t *testing.T) { writeFile(t, sidecarPath, resyncedSidecar) assertChainedInventory(t, bDir, fragmentPath, sidecarPath, lineUUIDB, lineUUIDA, observer.Port()) - removedSidecar := `{"inbounds":[],"schema":"lattice.singbox-metadata.v2"} -` - removeDoc := bindE2EDocument("remove", basename, digestText(fragment), nil, &removedSidecar) + removeDoc := bindE2EDocument("remove", basename, digestText(fragment), nil, lineUUIDB, "source-b", &targetLineUUID, nil) applyAndResolve(t, m, marshalE2EDocument(t, removeDoc), "remove-task", "remove-lease") if _, err := os.Stat(fragmentPath); !os.IsNotExist(err) { t.Fatalf("removed fragment still exists: %v", err) @@ -280,28 +279,45 @@ func TestLinechainE2EActiveHelper(t *testing.T) { type e2eDocument map[string]any -func bindE2EDocument(operation, basename, previousFragment string, fragment, sidecar *string) e2eDocument { - d := e2eDocument{"version": 2, "operation": operation, "fragment_basename": basename} - if previousFragment != "" { - d["previous_fragment_sha256"] = previousFragment - } - if fragment != nil { - d["fragment"] = *fragment +type e2eSidecarPatch struct { + Schema, SourceLineUUID, SourceInboundTag string + ExpectedDownstreamLineUUID, DesiredDownstreamLineUUID *string +} + +func (p e2eSidecarPatch) MarshalJSON() ([]byte, error) { + type wire struct { + Schema string `json:"schema"` + SourceLineUUID string `json:"source_line_uuid"` + SourceInboundTag string `json:"source_inbound_tag"` + Expected *string `json:"expected_downstream_line_uuid"` + Desired *string `json:"desired_downstream_line_uuid"` } - if sidecar != nil { - d["sidecar"] = *sidecar + return json.Marshal(wire{p.Schema, p.SourceLineUUID, p.SourceInboundTag, p.ExpectedDownstreamLineUUID, p.DesiredDownstreamLineUUID}) +} + +func bindE2EDocument(operation, basename, previousFragment string, fragment *string, sourceUUID, sourceTag string, expected, desired *string) e2eDocument { + patch := e2eSidecarPatch{"lattice.singbox-linechain-sidecar-patch.v1", sourceUUID, sourceTag, expected, desired} + patchBytes, _ := json.Marshal(patch) + var previous *string + if previousFragment != "" { + previous = &previousFragment } - fragmentDigest, sidecarDigest := digestPointer(fragment), digestPointer(sidecar) - d["fragment_sha256"], d["sidecar_sha256"] = fragmentDigest, sidecarDigest - pair := []byte{} + var fragmentSHA *string if fragment != nil { - pair = append(pair, *fragment...) - } - pair = append(pair, 0) - if sidecar != nil { - pair = append(pair, *sidecar...) - } - d["combined_sha256"] = digestBytes(pair) + value := digestText(*fragment) + fragmentSHA = &value + } + patchSHA := digestBytes(patchBytes) + type artifact struct { + Schema string `json:"schema"` + Operation string `json:"operation"` + Basename string `json:"fragment_basename"` + Previous *string `json:"previous_fragment_sha256"` + Fragment *string `json:"fragment_sha256"` + Patch string `json:"sidecar_patch_sha256"` + } + artifactBytes, _ := json.Marshal(artifact{"lattice.singbox-linechain-artifact.v2", operation, basename, previous, fragmentSHA, patchSHA}) + d := e2eDocument{"version": 2, "durable_protocol": "linechain-e3-v2", "operation": operation, "fragment_basename": basename, "fragment": fragment, "sidecar_patch": patch, "previous_fragment_sha256": previous, "fragment_sha256": fragmentSHA, "sidecar_patch_sha256": patchSHA, "artifact_sha256": digestBytes(artifactBytes)} return d } diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 2f2c568..fb3ca51 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -14,6 +14,21 @@ printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || { probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")" e2e_root="$(mktemp -d "${TMPDIR:-/tmp}/lattice-linechain-e2e.XXXXXX")" +agent_bin="$e2e_root/lattice-agent" +server_dir="${LATTICE_SERVER_E2E_DIR:-}" +if [ -z "$server_dir" ]; then + for candidate in ../../lattice-server/.wt/worker3-task18-server ../../lattice-server; do + if [ -f "$candidate/go.mod" ] && grep -q 'module github.com/LatticeNet/lattice-server' "$candidate/go.mod"; then + server_dir="$(cd "$candidate" && pwd -P)" + break + fi + done +fi +[ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } +[ "$(git -C "$server_dir" rev-parse 'HEAD^{tree}')" = "67dc25bc6740657449b6c206cf537b22398bc289" ] || { + echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 + exit 1 +} cleanup() { status=$? trap - EXIT HUP INT TERM @@ -50,4 +65,11 @@ JSON "$bin" check -C "$probe" >/dev/null 2>&1 || { echo "sing-box 1.13.x config check failed" >&2; exit 1; } printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')" +go build -trimpath -o "$agent_bin" ./cmd/lattice-agent +( + cd "$server_dir" + LATTICE_AGENT_E2E_BIN="$agent_bin" LATTICE_SINGBOX_E2E_BIN="$bin" \ + go test -tags=linechain_lifecycle_e2e ./internal/server -run '^TestLineChainPersistentServerAgentLifecycleE2E$' -count=1 -v +) + LATTICE_SINGBOX_E2E_BIN="$bin" LATTICE_LINECHAIN_E2E_ROOT="$e2e_root" go test -tags=linechain_e2e ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From 4f3bafb1869f55c15ad050eafe5362f133e6124a Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 10:19:58 -0700 Subject: [PATCH 29/43] Keep the lifecycle gate pinned after adding its tagged test Compare every server path against the repaired production tree while excluding only the new build-tagged lifecycle test itself, so the gate remains immutable without rejecting its own committed harness. Constraint: Task18 may add only the tagged server test beyond the Task21 production tree Confidence: high Scope-risk: narrow Reversibility: clean Tested: Shell static parse and exact lifecycle command Not-tested: GitHub CI --- scripts/test-linechain-e2e.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index fb3ca51..5354d8a 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -25,7 +25,8 @@ if [ -z "$server_dir" ]; then done fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } -[ "$(git -C "$server_dir" rev-parse 'HEAD^{tree}')" = "67dc25bc6740657449b6c206cf537b22398bc289" ] || { +git -C "$server_dir" diff --quiet 67dc25bc6740657449b6c206cf537b22398bc289 HEAD -- . \ + ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 exit 1 } From d61d2f622ec8624f0760a7d2681fecc62ccd357c Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 10:48:41 -0700 Subject: [PATCH 30/43] Drive real traffic from the server-issued lifecycle authority The mandatory E2E command now builds the tagged helper binary and keeps the server-issued lifecycle as the sole authority for live traffic, recovery, and remove verification rather than falling back to a second hand-built document. Constraint: Existing production files remain frozen; only E2E harness surfaces changed Rejected: Separate agent-local document test | it split artifact authority from persistent server leasing Confidence: high Scope-risk: narrow Directive: Keep the exact official sing-box command and no-skip leak gates Tested: Exact persistent lifecycle command with official sing-box 1.13.18 Not-tested: Full GitHub CI --- cmd/lattice-agent/linechain_e2e_test.go | 27 +++++++++++++++++++++++++ scripts/test-linechain-e2e.sh | 6 +++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index 8e2dfe5..a3ee3d3 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -37,6 +37,7 @@ const ( linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER" + linechainE2ERecoveryResult = "LATTICE_LINECHAIN_E2E_RECOVERY_RESULT" ) var managedE2EProcesses = struct { @@ -247,6 +248,32 @@ func TestLinechainE2EApplyHelper(t *testing.T) { } } +// TestLinechainE2ERecoverHelper is a child-only startup recovery entry point. +// It writes the recovered result only after RequireRecovered has restored the +// old artifacts and restarted the managed B process. +func TestLinechainE2ERecoverHelper(t *testing.T) { + resultPath := os.Getenv(linechainE2ERecoveryResult) + if resultPath == "" { + return + } + m := openE2EManager(t, os.Getenv(linechainE2EBinEnv), os.Getenv(linechainE2ERootEnv), os.Getenv(linechainE2EConfigEnv), os.Getenv(linechainE2ESidecarEnv), filepath.Join(os.Getenv(linechainE2ERootEnv), "txn"), mustEnvPort(linechainE2EBPortEnv)) + defer m.Close() + var recovered model.TaskResult + if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { + recovered = result + return nil + }, "node-b"); err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(recovered) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(resultPath, raw, 0o600); err != nil { + t.Fatal(err) + } +} + // TestLinechainE2ERestartHelper is the fixed restart/active command used by the // Manager. It stops the prior B instance and starts a checked replacement. func TestLinechainE2ERestartHelper(t *testing.T) { diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 5354d8a..db70b43 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -15,6 +15,7 @@ printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || { probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")" e2e_root="$(mktemp -d "${TMPDIR:-/tmp}/lattice-linechain-e2e.XXXXXX")" agent_bin="$e2e_root/lattice-agent" +agent_test_bin="$e2e_root/lattice-agent.test" server_dir="${LATTICE_SERVER_E2E_DIR:-}" if [ -z "$server_dir" ]; then for candidate in ../../lattice-server/.wt/worker3-task18-server ../../lattice-server; do @@ -67,10 +68,9 @@ JSON printf '%s\n' "linechain E2E binary: $(printf '%s\n' "$version" | sed -n '1p')" go build -trimpath -o "$agent_bin" ./cmd/lattice-agent +go test -c -tags=linechain_e2e -o "$agent_test_bin" ./cmd/lattice-agent ( cd "$server_dir" - LATTICE_AGENT_E2E_BIN="$agent_bin" LATTICE_SINGBOX_E2E_BIN="$bin" \ + LATTICE_AGENT_E2E_BIN="$agent_bin" LATTICE_AGENT_E2E_TEST_BIN="$agent_test_bin" LATTICE_SINGBOX_E2E_BIN="$bin" \ go test -tags=linechain_lifecycle_e2e ./internal/server -run '^TestLineChainPersistentServerAgentLifecycleE2E$' -count=1 -v ) - -LATTICE_SINGBOX_E2E_BIN="$bin" LATTICE_LINECHAIN_E2E_ROOT="$e2e_root" go test -tags=linechain_e2e ./cmd/lattice-agent -run '^TestLinechainRealSingBoxE2E$' -count=1 -v From 6523b1cb58e155ac632899e7b041b22d8fd40d9e Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 10:53:55 -0700 Subject: [PATCH 31/43] Make the persistent lifecycle test observe real agent recovery Expose strict tagged-binary entrypoints for startup recovery, durable result resolution, journal cleanup, and runtime inventory so the server lifecycle cannot fabricate those transitions. Harden managed sing-box teardown around process groups and explicit reaping to make residual-process failures visible. Constraint: Task24 owns the server consumer and official binary lifecycle Constraint: This task may modify only the node-agent harness files Rejected: Keep synchronous fabricated server results | bypasses production recovery authority Rejected: Signal only the sing-box leader PID | can leak process-group children Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep the exported environment names stable across the tagged server harness Tested: go test ./cmd/lattice-agent -count=1 Tested: go test -tags=linechain_e2e ./cmd/lattice-agent -run ^TestKillManagedSingBox -count=1 Tested: go test -c -tags=linechain_e2e ./cmd/lattice-agent Tested: go vet ./cmd/lattice-agent and go vet -tags=linechain_e2e ./cmd/lattice-agent Tested: gofmt and git diff --check Not-tested: Official sing-box server lifecycle is owned by Task24 --- cmd/lattice-agent/linechain_e2e_test.go | 271 +++++++++++++++++++----- 1 file changed, 222 insertions(+), 49 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index a3ee3d3..b84a86a 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -29,15 +29,17 @@ import ( ) const ( - linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN" - linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT" - linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR" - linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR" - linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT" - linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" - linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" - linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER" - linechainE2ERecoveryResult = "LATTICE_LINECHAIN_E2E_RECOVERY_RESULT" + linechainE2EBinEnv = "LATTICE_SINGBOX_E2E_BIN" + linechainE2ERootEnv = "LATTICE_LINECHAIN_E2E_ROOT" + linechainE2EConfigEnv = "LATTICE_LINECHAIN_E2E_CONFIG_DIR" + linechainE2ESidecarEnv = "LATTICE_LINECHAIN_E2E_SIDECAR" + linechainE2EBPortEnv = "LATTICE_LINECHAIN_E2E_B_PORT" + linechainE2ETaskEnv = "LATTICE_LINECHAIN_E2E_TASK" + linechainE2ELeaseEnv = "LATTICE_LINECHAIN_E2E_LEASE" + linechainE2ECrashMarkerEnv = "LATTICE_LINECHAIN_E2E_CRASH_MARKER" + linechainE2ERecoveryResult = "LATTICE_LINECHAIN_E2E_RECOVERY_RESULT" + linechainE2EResolveResult = "LATTICE_LINECHAIN_E2E_RESOLVE_RESULT" + linechainE2EInventoryResult = "LATTICE_LINECHAIN_E2E_INVENTORY_RESULT" ) var managedE2EProcesses = struct { @@ -256,32 +258,77 @@ func TestLinechainE2ERecoverHelper(t *testing.T) { if resultPath == "" { return } - m := openE2EManager(t, os.Getenv(linechainE2EBinEnv), os.Getenv(linechainE2ERootEnv), os.Getenv(linechainE2EConfigEnv), os.Getenv(linechainE2ESidecarEnv), filepath.Join(os.Getenv(linechainE2ERootEnv), "txn"), mustEnvPort(linechainE2EBPortEnv)) + root := mustAbsoluteEnv(t, linechainE2ERootEnv) + consumeE2ECrashMarker(t, mustAbsoluteEnv(t, linechainE2ECrashMarkerEnv)) + m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv)) defer m.Close() var recovered model.TaskResult + called := false if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { + called = true recovered = result return nil }, "node-b"); err != nil { t.Fatal(err) } - raw, err := json.Marshal(recovered) + if !called || recovered.TaskID == "" || recovered.LeaseID == "" { + t.Fatalf("recovery emitted no exact task result: %+v", recovered) + } + writeE2EJSON(t, resultPath, recovered) +} + +// TestLinechainE2EResolveHelper converts a successful helper run into the +// stable durable result, cleans its journal, and only then publishes JSON. +func TestLinechainE2EResolveHelper(t *testing.T) { + resultPath := os.Getenv(linechainE2EResolveResult) + if resultPath == "" { + return + } + root := mustAbsoluteEnv(t, linechainE2ERootEnv) + taskID := mustEnv(t, linechainE2ETaskEnv) + leaseID := mustEnv(t, linechainE2ELeaseEnv) + m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv)) + defer m.Close() + result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}) if err != nil { t.Fatal(err) } - if err := os.WriteFile(resultPath, raw, 0o600); err != nil { + if err := m.Cleanup(taskID, leaseID); err != nil { + t.Fatal(err) + } + writeE2EJSON(t, resultPath, result) +} + +// TestLinechainE2EInventoryHelper publishes the inventory discovered from the +// actual post-restart config directory and semantic sidecar. +func TestLinechainE2EInventoryHelper(t *testing.T) { + resultPath := os.Getenv(linechainE2EInventoryResult) + if resultPath == "" { + return + } + configDir := mustAbsoluteEnv(t, linechainE2EConfigEnv) + paths, err := filepath.Glob(filepath.Join(configDir, "*.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("discover config files: paths=%v err=%v", paths, err) + } + inv, err := singboxdiscover.DiscoverRuntimeFiles("node-b", paths, mustAbsoluteEnv(t, linechainE2ESidecarEnv)) + if err != nil { t.Fatal(err) } + writeE2EJSON(t, resultPath, inv) } // TestLinechainE2ERestartHelper is the fixed restart/active command used by the // Manager. It stops the prior B instance and starts a checked replacement. func TestLinechainE2ERestartHelper(t *testing.T) { - root := os.Getenv(linechainE2ERootEnv) - if root == "" { + if os.Getenv(linechainE2ERootEnv) == "" { return } + root := mustAbsoluteEnv(t, linechainE2ERootEnv) if marker := os.Getenv(linechainE2ECrashMarkerEnv); marker != "" { + if !filepath.IsAbs(marker) { + t.Fatalf("%s must be absolute", linechainE2ECrashMarkerEnv) + } if err := os.WriteFile(marker, []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600); err != nil { t.Fatal(err) } @@ -289,11 +336,71 @@ func TestLinechainE2ERestartHelper(t *testing.T) { time.Sleep(time.Hour) } } - if err := restartManagedSingBox(os.Getenv(linechainE2EBinEnv), root, "b", os.Getenv(linechainE2EConfigEnv), mustEnvPort(linechainE2EBPortEnv)); err != nil { + if err := restartManagedSingBox(mustExecutableEnv(t, linechainE2EBinEnv), root, "b", mustAbsoluteEnv(t, linechainE2EConfigEnv), mustEnvPort(linechainE2EBPortEnv)); err != nil { + t.Fatal(err) + } +} + +func mustEnv(t *testing.T, key string) string { + t.Helper() + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + t.Fatalf("%s is required", key) + } + return value +} + +func mustAbsoluteEnv(t *testing.T, key string) string { + t.Helper() + value := mustEnv(t, key) + if !filepath.IsAbs(value) { + t.Fatalf("%s must be absolute", key) + } + return value +} + +func mustExecutableEnv(t *testing.T, key string) string { + t.Helper() + value := mustAbsoluteEnv(t, key) + info, err := os.Stat(value) + if err != nil || info.IsDir() || info.Mode()&0o111 == 0 { + t.Fatalf("%s must be executable: %v", key, err) + } + return value +} + +func writeE2EJSON(t *testing.T, path string, value any) { + t.Helper() + if !filepath.IsAbs(path) { + t.Fatalf("result path must be absolute: %s", path) + } + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, raw, 0o600); err != nil { t.Fatal(err) } } +func consumeE2ECrashMarker(t *testing.T, path string) { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read one-shot crash marker: %v", err) + } + pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || pid <= 1 { + t.Fatalf("invalid one-shot crash marker %q", raw) + } + if err := os.Remove(path); err != nil { + t.Fatalf("consume one-shot crash marker: %v", err) + } + if err := os.Unsetenv(linechainE2ECrashMarkerEnv); err != nil { + t.Fatalf("clear one-shot crash marker env: %v", err) + } +} + func TestLinechainE2EActiveHelper(t *testing.T) { root := os.Getenv(linechainE2ERootEnv) if root == "" { @@ -584,7 +691,11 @@ func startManagedSingBox(t *testing.T, bin, root, name, configDir string, port i if err := restartManagedSingBox(bin, root, name, configDir, port); err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = killManagedSingBox(root, name) }) + t.Cleanup(func() { + if err := killManagedSingBox(root, name); err != nil { + t.Errorf("cleanup managed sing-box %s: %v", name, err) + } + }) } func restartManagedSingBox(bin, root, name, configDir string, port int) error { @@ -611,17 +722,27 @@ func restartManagedSingBox(bin, root, name, configDir string, port int) error { _ = logFile.Close() if err := os.WriteFile(filepath.Join(root, name+".pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + exited := waitManagedProcess(cmd.Process.Pid, done, 2*time.Second) + managedE2EProcesses.Lock() + delete(managedE2EProcesses.done, cmd.Process.Pid) + managedE2EProcesses.Unlock() + if !exited { + return fmt.Errorf("write %s pid file: %w; process group %d remained after SIGKILL", name, err, cmd.Process.Pid) + } return err } if err := waitForPort(port, 8*time.Second); err != nil { - _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + cleanupErr := killManagedSingBox(root, name) logBytes, _ := os.ReadFile(logPath) + if cleanupErr != nil { + return fmt.Errorf("start %s: %w: %s; cleanup: %v", name, err, logBytes, cleanupErr) + } return fmt.Errorf("start %s: %w: %s", name, err, logBytes) } return nil } -func killManagedSingBox(root, name string) error { +func killManagedSingBox(root, name string) (retErr error) { pidPath := filepath.Join(root, name+".pid") raw, err := os.ReadFile(pidPath) if os.IsNotExist(err) { @@ -632,52 +753,104 @@ func killManagedSingBox(root, name string) error { } pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) if err != nil || pid <= 1 { - return fmt.Errorf("invalid %s pid %q", name, raw) - } - if err := syscall.Kill(pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH { - return err + parseErr := fmt.Errorf("invalid %s pid %q", name, raw) + if removeErr := os.Remove(pidPath); removeErr != nil && !os.IsNotExist(removeErr) { + return fmt.Errorf("%v; remove invalid pid file: %w", parseErr, removeErr) + } + return parseErr } managedE2EProcesses.Lock() done := managedE2EProcesses.done[pid] managedE2EProcesses.Unlock() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if managedProcessExited(pid, done) { - managedE2EProcesses.Lock() - delete(managedE2EProcesses.done, pid) - managedE2EProcesses.Unlock() - _ = os.Remove(pidPath) - return nil + defer func() { + managedE2EProcesses.Lock() + delete(managedE2EProcesses.done, pid) + managedE2EProcesses.Unlock() + if err := os.Remove(pidPath); err != nil && !os.IsNotExist(err) { + if retErr == nil { + retErr = fmt.Errorf("remove %s pid file: %w", name, err) + } else { + retErr = fmt.Errorf("%v; remove %s pid file: %w", retErr, name, err) + } } - time.Sleep(20 * time.Millisecond) + }() + if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH { + return err + } + if waitManagedProcess(pid, done, 2*time.Second) { + return nil } - if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { + if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { return err } - killDeadline := time.Now().Add(2 * time.Second) - for time.Now().Before(killDeadline) { - if managedProcessExited(pid, done) { - managedE2EProcesses.Lock() - delete(managedE2EProcesses.done, pid) - managedE2EProcesses.Unlock() - _ = os.Remove(pidPath) - return nil - } - time.Sleep(20 * time.Millisecond) + if waitManagedProcess(pid, done, 2*time.Second) { + return nil } return fmt.Errorf("%s process %d remained after SIGKILL", name, pid) } -func managedProcessExited(pid int, done <-chan error) bool { - if done != nil { - select { - case <-done: +func waitManagedProcess(pid int, done <-chan error, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if done != nil { + select { + case <-done: + return true + default: + } + } else if syscall.Kill(-pid, 0) == syscall.ESRCH { return true - default: - return false } + time.Sleep(20 * time.Millisecond) + } + return false +} + +func TestKillManagedSingBoxTerminatesProcessGroupAndClearsState(t *testing.T) { + root := t.TempDir() + cmd := exec.Command("sh", "-c", "trap 'exit 0' TERM; sleep 60 & wait") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + managedE2EProcesses.Lock() + managedE2EProcesses.done[cmd.Process.Pid] = done + managedE2EProcesses.Unlock() + go func() { done <- cmd.Wait() }() + pidPath := filepath.Join(root, "test.pid") + if err := os.WriteFile(pidPath, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { + t.Fatal(err) + } + if err := killManagedSingBox(root, "test"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(pidPath); !os.IsNotExist(err) { + t.Fatalf("pid file survived cleanup: %v", err) + } + managedE2EProcesses.Lock() + _, tracked := managedE2EProcesses.done[cmd.Process.Pid] + managedE2EProcesses.Unlock() + if tracked { + t.Fatal("managed process survived in reap map") + } + if err := syscall.Kill(-cmd.Process.Pid, 0); err != syscall.ESRCH { + t.Fatalf("managed process group survived cleanup: %v", err) + } +} + +func TestKillManagedSingBoxRejectsAndRemovesInvalidPID(t *testing.T) { + root := t.TempDir() + pidPath := filepath.Join(root, "invalid.pid") + if err := os.WriteFile(pidPath, []byte("not-a-pid"), 0o600); err != nil { + t.Fatal(err) + } + if err := killManagedSingBox(root, "invalid"); err == nil || !strings.Contains(err.Error(), "invalid invalid pid") { + t.Fatalf("invalid pid error = %v", err) + } + if _, err := os.Stat(pidPath); !os.IsNotExist(err) { + t.Fatalf("invalid pid file survived cleanup: %v", err) } - return syscall.Kill(pid, 0) == syscall.ESRCH } func killMarkerProcess(t *testing.T, marker string) { From ce8bad3b7936545f0940331066e0ead4bdfa3ad1 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 10:54:55 -0700 Subject: [PATCH 32/43] Keep the production-tree pin while admitting the tagged lifecycle consumer Allow the exact Task24 tagged network E2E file alongside the existing tagged lifecycle file so the harness can run without weakening its production source comparison. Constraint: Task24 adds one named tagged server test outside the frozen production tree Rejected: Exclude all E2E files by wildcard | would permit unrelated source drift Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep this allowlist exact; do not exclude production server paths Tested: sh -n scripts/test-linechain-e2e.sh Tested: frozen server diff check with both exact path exclusions Tested: git diff --check Not-tested: Official binary lifecycle remains Task24 --- scripts/test-linechain-e2e.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index db70b43..7343b07 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -27,7 +27,8 @@ if [ -z "$server_dir" ]; then fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } git -C "$server_dir" diff --quiet 67dc25bc6740657449b6c206cf537b22398bc289 HEAD -- . \ - ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' || { + ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ + ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 exit 1 } From a183a39d7a3943bc3dd0b7cac960736cb136c9a9 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:10:06 -0700 Subject: [PATCH 33/43] Make the lifecycle leak gate own every server runtime process group Place the server lifecycle beneath one script-owned absolute runtime root and clean leaked work by unique process group rather than command leader PID. The gate now verifies both group disappearance and an empty final runtime scan. Constraint: Server A/B/client paths previously escaped the script build root Constraint: Task25 owns only scripts/test-linechain-e2e.sh Rejected: Scan only the agent build root | misses server t.TempDir processes Rejected: Kill matching leader PIDs | can leave process-group children alive Confidence: high Scope-risk: narrow Reversibility: clean Directive: Task24 must keep all lifecycle runtime paths under LATTICE_LINECHAIN_E2E_RUNTIME_ROOT Tested: sh -n scripts/test-linechain-e2e.sh Tested: git diff --check Not-tested: Official binary lifecycle awaits Task24 adoption of the runtime-root env --- scripts/test-linechain-e2e.sh | 51 +++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 7343b07..870b606 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -14,6 +14,8 @@ printf '%s\n' "$version" | grep -Eq '^sing-box version 1\.13\.[0-9]+' || { probe="$(mktemp -d "${TMPDIR:-/tmp}/lattice-sing-box-check.XXXXXX")" e2e_root="$(mktemp -d "${TMPDIR:-/tmp}/lattice-linechain-e2e.XXXXXX")" +runtime_root="$e2e_root/runtime" +mkdir -m 700 "$runtime_root" agent_bin="$e2e_root/lattice-agent" agent_test_bin="$e2e_root/lattice-agent.test" server_dir="${LATTICE_SERVER_E2E_DIR:-}" @@ -34,15 +36,41 @@ git -C "$server_dir" diff --quiet 67dc25bc6740657449b6c206cf537b22398bc289 HEAD } cleanup() { status=$? - trap - EXIT HUP INT TERM - if ps -axo command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep >/dev/null; then - echo "linechain E2E leaked process for $e2e_root" >&2 - ps -axo pid=,ppid=,command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep >&2 || true - ps -axo pid=,command= | grep -F "$e2e_root" | grep -E 'sing-box|lattice-agent\.test' | grep -v grep | while read -r leaked_pid _; do - kill -TERM "-$leaked_pid" 2>/dev/null || kill -TERM "$leaked_pid" 2>/dev/null || true - sleep 0.1 - kill -KILL "-$leaked_pid" 2>/dev/null || kill -KILL "$leaked_pid" 2>/dev/null || true - done + trap - EXIT HUP INT TERM + process_snapshot="$e2e_root/processes" + pgid_snapshot="$e2e_root/pgids" + ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' + index($0, "awk -v runtime=") == 0 && (index($0, runtime) || index($0, agent_test)) { print } + ' >"$process_snapshot" + if [ -s "$process_snapshot" ]; then + echo "linechain E2E leaked process for runtime root $runtime_root" >&2 + cat "$process_snapshot" >&2 + awk '{print $2}' "$process_snapshot" | sort -n -u >"$pgid_snapshot" + while read -r leaked_pgid; do + [ -n "$leaked_pgid" ] || continue + kill -TERM "-$leaked_pgid" 2>/dev/null || true + done <"$pgid_snapshot" + sleep 0.2 + while read -r leaked_pgid; do + [ -n "$leaked_pgid" ] || continue + kill -KILL "-$leaked_pgid" 2>/dev/null || true + done <"$pgid_snapshot" + sleep 0.2 + while read -r leaked_pgid; do + [ -n "$leaked_pgid" ] || continue + if kill -0 "-$leaked_pgid" 2>/dev/null; then + echo "linechain E2E process group $leaked_pgid survived cleanup" >&2 + status=1 + fi + done <"$pgid_snapshot" + ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' + index($0, "awk -v runtime=") == 0 && (index($0, runtime) || index($0, agent_test)) { print } + ' >"$process_snapshot" + if [ -s "$process_snapshot" ]; then + echo "linechain E2E runtime process scan remained non-empty" >&2 + cat "$process_snapshot" >&2 + status=1 + fi status=1 fi find "$probe" "$e2e_root" -type f -delete 2>/dev/null || true @@ -52,9 +80,9 @@ cleanup() { trap cleanup EXIT HUP INT TERM # Prove the root-tagged process detector before relying on it for the real run. -sh -c 'trap : TERM; while :; do sleep 1; done' "$e2e_root" & +sh -c 'trap : TERM; while :; do sleep 1; done' "$runtime_root" & detector_pid=$! -ps -axo command= | grep -F "$e2e_root" | grep -F 'while :' >/dev/null || { +ps -o command= -p "$detector_pid" | grep -F "$runtime_root" | grep -F 'while :' >/dev/null || { kill "$detector_pid" 2>/dev/null || true wait "$detector_pid" 2>/dev/null || true echo "linechain E2E leak detector self-test failed" >&2 @@ -73,5 +101,6 @@ go test -c -tags=linechain_e2e -o "$agent_test_bin" ./cmd/lattice-agent ( cd "$server_dir" LATTICE_AGENT_E2E_BIN="$agent_bin" LATTICE_AGENT_E2E_TEST_BIN="$agent_test_bin" LATTICE_SINGBOX_E2E_BIN="$bin" \ + LATTICE_LINECHAIN_E2E_RUNTIME_ROOT="$runtime_root" \ go test -tags=linechain_lifecycle_e2e ./internal/server -run '^TestLineChainPersistentServerAgentLifecycleE2E$' -count=1 -v ) From 8df7b2a33e0f7d1ec4695950ca98e3305875b5b1 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:11:17 -0700 Subject: [PATCH 34/43] Require full process-group disappearance before harness cleanup succeeds Track leader reaping separately from process-group liveness so a TERM-exiting leader cannot hide a surviving child. Cleanup now falls through to SIGKILL until both the owned leader is reaped and the entire PGID reports ESRCH. Constraint: TERM-ignoring sing-box descendants can outlive a reaped group leader Constraint: Task26 owns only the tagged node-agent harness file Rejected: Treat cmd.Wait completion as group completion | leaves descendant processes alive Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep leader-reap and PGID-disappearance checks independent Tested: go test -race -tags=linechain_e2e ./cmd/lattice-agent -run ^TestKillManagedSingBox -count=1 Tested: go vet -tags=linechain_e2e ./cmd/lattice-agent Tested: gofmt and git diff --check Not-tested: Official binary lifecycle remains Task24 --- cmd/lattice-agent/linechain_e2e_test.go | 30 ++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index b84a86a..d856cf7 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -722,7 +722,8 @@ func restartManagedSingBox(bin, root, name, configDir string, port int) error { _ = logFile.Close() if err := os.WriteFile(filepath.Join(root, name+".pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) - exited := waitManagedProcess(cmd.Process.Pid, done, 2*time.Second) + leaderReaped := false + exited := waitManagedProcess(cmd.Process.Pid, done, &leaderReaped, 2*time.Second) managedE2EProcesses.Lock() delete(managedE2EProcesses.done, cmd.Process.Pid) managedE2EProcesses.Unlock() @@ -777,28 +778,30 @@ func killManagedSingBox(root, name string) (retErr error) { if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && err != syscall.ESRCH { return err } - if waitManagedProcess(pid, done, 2*time.Second) { + leaderReaped := done == nil + if waitManagedProcess(pid, done, &leaderReaped, 2*time.Second) { return nil } if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { return err } - if waitManagedProcess(pid, done, 2*time.Second) { + if waitManagedProcess(pid, done, &leaderReaped, 2*time.Second) { return nil } return fmt.Errorf("%s process %d remained after SIGKILL", name, pid) } -func waitManagedProcess(pid int, done <-chan error, timeout time.Duration) bool { +func waitManagedProcess(pid int, done <-chan error, leaderReaped *bool, timeout time.Duration) bool { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if done != nil { + if !*leaderReaped && done != nil { select { case <-done: - return true + *leaderReaped = true default: } - } else if syscall.Kill(-pid, 0) == syscall.ESRCH { + } + if *leaderReaped && syscall.Kill(-pid, 0) == syscall.ESRCH { return true } time.Sleep(20 * time.Millisecond) @@ -808,7 +811,8 @@ func waitManagedProcess(pid int, done <-chan error, timeout time.Duration) bool func TestKillManagedSingBoxTerminatesProcessGroupAndClearsState(t *testing.T) { root := t.TempDir() - cmd := exec.Command("sh", "-c", "trap 'exit 0' TERM; sleep 60 & wait") + childPIDPath := filepath.Join(root, "child.pid") + cmd := exec.Command("sh", "-c", `trap 'exit 0' TERM; sh -c 'trap "" TERM; while :; do sleep 1; done' & echo $! > "$1"; wait`, "managed-test", childPIDPath) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := cmd.Start(); err != nil { t.Fatal(err) @@ -818,6 +822,16 @@ func TestKillManagedSingBoxTerminatesProcessGroupAndClearsState(t *testing.T) { managedE2EProcesses.done[cmd.Process.Pid] = done managedE2EProcesses.Unlock() go func() { done <- cmd.Wait() }() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(childPIDPath); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("TERM-ignoring child did not start") + } + time.Sleep(10 * time.Millisecond) + } pidPath := filepath.Join(root, "test.pid") if err := os.WriteFile(pidPath, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { t.Fatal(err) From ca912a87c9da60475a88ef5592d9368319842cab Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:14:15 -0700 Subject: [PATCH 35/43] Make tagged lifecycle results follow the production durable handoff Journal the exact leased E3 task before execution, persist and confirm the terminal result before transaction cleanup, and export only a result reread from the durable outbox. Also reject dirty or untracked server worktrees before the canonical lifecycle begins. Constraint: Production ordering is outbox Begin before execution and durable Complete before linechain Cleanup Constraint: Task27 owns only the two tagged node harness files Rejected: Export callback memory before durable outbox write | loses results across the cleanup crash window Rejected: Trust HEAD while allowing dirty tagged files | executes unreviewed lifecycle code Confidence: high Scope-risk: moderate Reversibility: clean Directive: Server Task24 must invoke BeginHelper before the exact leased script and reuse the same outbox directory Tested: go test -race -tags=linechain_e2e ./cmd/lattice-agent -run ^TestLinechainE2EDurableResultSurvivesOutboxReopen$ -count=1 Tested: go test -c -tags=linechain_e2e ./cmd/lattice-agent Tested: go vet -tags=linechain_e2e ./cmd/lattice-agent Tested: sh -n scripts/test-linechain-e2e.sh; gofmt; git diff --check Not-tested: Official server-issued lifecycle awaits Task24 interface adoption --- cmd/lattice-agent/linechain_e2e_test.go | 112 +++++++++++++++++++++--- scripts/test-linechain-e2e.sh | 1 + 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index d856cf7..7bef0ad 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -25,6 +25,7 @@ import ( "github.com/LatticeNet/lattice-node-agent/internal/linechain" "github.com/LatticeNet/lattice-node-agent/internal/singboxdiscover" + "github.com/LatticeNet/lattice-node-agent/internal/taskoutbox" "github.com/LatticeNet/lattice-sdk/model" ) @@ -40,8 +41,40 @@ const ( linechainE2ERecoveryResult = "LATTICE_LINECHAIN_E2E_RECOVERY_RESULT" linechainE2EResolveResult = "LATTICE_LINECHAIN_E2E_RESOLVE_RESULT" linechainE2EInventoryResult = "LATTICE_LINECHAIN_E2E_INVENTORY_RESULT" + linechainE2EOutboxEnv = "LATTICE_LINECHAIN_E2E_OUTBOX" + linechainE2ETaskJSONEnv = "LATTICE_LINECHAIN_E2E_TASK_JSON" + linechainE2EBeginResultEnv = "LATTICE_LINECHAIN_E2E_BEGIN_RESULT" ) +// TestLinechainE2EBeginHelper durably records the exact server lease before +// its script is allowed to execute. +func TestLinechainE2EBeginHelper(t *testing.T) { + resultPath := os.Getenv(linechainE2EBeginResultEnv) + if resultPath == "" { + return + } + var task model.Task + raw, err := os.ReadFile(mustAbsoluteEnv(t, linechainE2ETaskJSONEnv)) + if err != nil || json.Unmarshal(raw, &task) != nil { + t.Fatalf("decode exact leased task: %v", err) + } + outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv)) + if err != nil { + t.Fatal(err) + } + defer outbox.Close() + committed, err := outbox.BeginWithProtocol(task, "linechain-e3-v2") + if err != nil { + t.Fatal(err) + } + if committed { + if err := outbox.ConfirmDurability(); err != nil { + t.Fatal(err) + } + } + writeE2EJSON(t, resultPath, map[string]any{"committed": committed, "task_id": task.ID, "lease_id": task.LeaseID}) +} + var managedE2EProcesses = struct { sync.Mutex done map[int]<-chan error @@ -262,19 +295,25 @@ func TestLinechainE2ERecoverHelper(t *testing.T) { consumeE2ECrashMarker(t, mustAbsoluteEnv(t, linechainE2ECrashMarkerEnv)) m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv)) defer m.Close() - var recovered model.TaskResult - called := false - if err := m.RequireRecovered(context.Background(), func(result model.TaskResult) error { - called = true - recovered = result - return nil - }, "node-b"); err != nil { + outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv)) + if err != nil { t.Fatal(err) } - if !called || recovered.TaskID == "" || recovered.LeaseID == "" { - t.Fatalf("recovery emitted no exact task result: %+v", recovered) + defer outbox.Close() + authority, err := captureLinechainAuthority(m, outbox) + if err != nil { + t.Fatal(err) + } + if err := m.RequireRecoveredAuthorized(context.Background(), func(result model.TaskResult) error { + committed, completeErr := outbox.Complete(result) + if completeErr != nil && !committed { + return completeErr + } + return outbox.ConfirmDurability() + }, "node-b", authority); err != nil { + t.Fatal(err) } - writeE2EJSON(t, resultPath, recovered) + writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox)) } // TestLinechainE2EResolveHelper converts a successful helper run into the @@ -289,14 +328,65 @@ func TestLinechainE2EResolveHelper(t *testing.T) { leaseID := mustEnv(t, linechainE2ELeaseEnv) m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv)) defer m.Close() + outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv)) + if err != nil { + t.Fatal(err) + } + defer outbox.Close() result, err := m.ResolveAfterRun(context.Background(), model.Task{ID: taskID, LeaseID: leaseID}, model.TaskResult{TaskID: taskID, LeaseID: leaseID, ExitCode: 0, FinishedAt: time.Now().UTC()}) if err != nil { t.Fatal(err) } + committed, err := outbox.Complete(result) + if err != nil && !committed { + t.Fatal(err) + } + if err := outbox.ConfirmDurability(); err != nil { + t.Fatal(err) + } if err := m.Cleanup(taskID, leaseID); err != nil { t.Fatal(err) } - writeE2EJSON(t, resultPath, result) + writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox)) +} + +func pendingE2EResult(t *testing.T, outbox *taskoutbox.Store) model.TaskResult { + t.Helper() + entries, err := outbox.Pending() + if err != nil || len(entries) != 1 || entries[0].Result == nil { + t.Fatalf("durable result handoff: entries=%+v err=%v", entries, err) + } + return *entries[0].Result +} + +func TestLinechainE2EDurableResultSurvivesOutboxReopen(t *testing.T) { + dir := filepath.Join(t.TempDir(), "outbox") + task := model.Task{ID: "task-reopen", LeaseID: "lease-reopen", Script: "# lattice-linechain-e3-v2\nexact"} + outbox, err := taskoutbox.Open(dir) + if err != nil { + t.Fatal(err) + } + if committed, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil || !committed { + t.Fatalf("begin: committed=%v err=%v", committed, err) + } + result := model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0, FinishedAt: time.Now().UTC()} + if committed, err := outbox.Complete(result); err != nil || !committed { + t.Fatalf("complete: committed=%v err=%v", committed, err) + } + if err := outbox.ConfirmDurability(); err != nil { + t.Fatal(err) + } + if err := outbox.Close(); err != nil { + t.Fatal(err) + } + reopened, err := taskoutbox.Open(dir) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if got := pendingE2EResult(t, reopened); !sameTaskResult(got, result) { + t.Fatalf("reopened result changed: got=%+v want=%+v", got, result) + } } // TestLinechainE2EInventoryHelper publishes the inventory discovered from the diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 870b606..a90916e 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -28,6 +28,7 @@ if [ -z "$server_dir" ]; then done fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } +[ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } git -C "$server_dir" diff --quiet 67dc25bc6740657449b6c206cf537b22398bc289 HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { From 9ae0060cd87e5b9110171c50a4ddcbd7510bd087 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:18:20 -0700 Subject: [PATCH 36/43] Preserve exact E3 results through recovery failure and acknowledgement Select durable results by exact task and lease, remove only the server-acknowledged entry, and prove a missing outbox Begin cannot clean the terminal transaction. Authorized recovery now persists the exact result before journal cleanup and survives unrelated pending entries. Constraint: Server replays acknowledge one exact task result at a time Constraint: Task28 owns only tagged node-agent harness files Rejected: Require a singleton Pending list | breaks create/remove sequences in one outbox Rejected: Cleanup after an in-memory recovery result | loses authority when outbox completion fails Confidence: high Scope-risk: moderate Reversibility: clean Directive: Task24 must invoke AckHelper only after a successful exact server replay Tested: go test -race -tags=linechain_e2e ./cmd/lattice-agent -run ^TestLinechainE2E(RecoveryDoesNotCleanupBeforeDurableResult|DurableResultSurvivesOutboxReopen)$ -count=1 Tested: go test -c -tags=linechain_e2e ./cmd/lattice-agent Tested: go vet -tags=linechain_e2e ./cmd/lattice-agent Tested: gofmt and git diff --check Not-tested: Official binary lifecycle awaits Task24 AckHelper adoption --- cmd/lattice-agent/linechain_e2e_test.go | 131 ++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 6 deletions(-) diff --git a/cmd/lattice-agent/linechain_e2e_test.go b/cmd/lattice-agent/linechain_e2e_test.go index 7bef0ad..407ff01 100644 --- a/cmd/lattice-agent/linechain_e2e_test.go +++ b/cmd/lattice-agent/linechain_e2e_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/binary" "encoding/hex" "encoding/json" @@ -44,6 +45,7 @@ const ( linechainE2EOutboxEnv = "LATTICE_LINECHAIN_E2E_OUTBOX" linechainE2ETaskJSONEnv = "LATTICE_LINECHAIN_E2E_TASK_JSON" linechainE2EBeginResultEnv = "LATTICE_LINECHAIN_E2E_BEGIN_RESULT" + linechainE2EAckResultEnv = "LATTICE_LINECHAIN_E2E_ACK_RESULT" ) // TestLinechainE2EBeginHelper durably records the exact server lease before @@ -75,6 +77,26 @@ func TestLinechainE2EBeginHelper(t *testing.T) { writeE2EJSON(t, resultPath, map[string]any{"committed": committed, "task_id": task.ID, "lease_id": task.LeaseID}) } +// TestLinechainE2EAckHelper durably removes only the exact result acknowledged +// by the server after its successful replay response. +func TestLinechainE2EAckHelper(t *testing.T) { + resultPath := os.Getenv(linechainE2EAckResultEnv) + if resultPath == "" { + return + } + taskID, leaseID := mustEnv(t, linechainE2ETaskEnv), mustEnv(t, linechainE2ELeaseEnv) + outbox, err := taskoutbox.Open(mustAbsoluteEnv(t, linechainE2EOutboxEnv)) + if err != nil { + t.Fatal(err) + } + defer outbox.Close() + entry := pendingE2EEntry(t, outbox, taskID, leaseID) + if err := outbox.Remove(entry); err != nil { + t.Fatal(err) + } + writeE2EJSON(t, resultPath, map[string]string{"task_id": taskID, "lease_id": leaseID}) +} + var managedE2EProcesses = struct { sync.Mutex done map[int]<-chan error @@ -292,6 +314,7 @@ func TestLinechainE2ERecoverHelper(t *testing.T) { return } root := mustAbsoluteEnv(t, linechainE2ERootEnv) + taskID, leaseID := mustEnv(t, linechainE2ETaskEnv), mustEnv(t, linechainE2ELeaseEnv) consumeE2ECrashMarker(t, mustAbsoluteEnv(t, linechainE2ECrashMarkerEnv)) m := openE2EManager(t, mustExecutableEnv(t, linechainE2EBinEnv), root, mustAbsoluteEnv(t, linechainE2EConfigEnv), mustAbsoluteEnv(t, linechainE2ESidecarEnv), filepath.Join(root, "txn"), mustEnvPort(linechainE2EBPortEnv)) defer m.Close() @@ -313,7 +336,7 @@ func TestLinechainE2ERecoverHelper(t *testing.T) { }, "node-b", authority); err != nil { t.Fatal(err) } - writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox)) + writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox, taskID, leaseID)) } // TestLinechainE2EResolveHelper converts a successful helper run into the @@ -347,16 +370,27 @@ func TestLinechainE2EResolveHelper(t *testing.T) { if err := m.Cleanup(taskID, leaseID); err != nil { t.Fatal(err) } - writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox)) + writeE2EJSON(t, resultPath, pendingE2EResult(t, outbox, taskID, leaseID)) } -func pendingE2EResult(t *testing.T, outbox *taskoutbox.Store) model.TaskResult { +func pendingE2EEntry(t *testing.T, outbox *taskoutbox.Store, taskID, leaseID string) taskoutbox.Entry { t.Helper() entries, err := outbox.Pending() - if err != nil || len(entries) != 1 || entries[0].Result == nil { + if err != nil { t.Fatalf("durable result handoff: entries=%+v err=%v", entries, err) } - return *entries[0].Result + for _, entry := range entries { + if entry.Task.ID == taskID && entry.Task.LeaseID == leaseID && entry.Result != nil { + return entry + } + } + t.Fatalf("durable result %s/%s not found: %+v", taskID, leaseID, entries) + return taskoutbox.Entry{} +} + +func pendingE2EResult(t *testing.T, outbox *taskoutbox.Store, taskID, leaseID string) model.TaskResult { + t.Helper() + return *pendingE2EEntry(t, outbox, taskID, leaseID).Result } func TestLinechainE2EDurableResultSurvivesOutboxReopen(t *testing.T) { @@ -384,9 +418,94 @@ func TestLinechainE2EDurableResultSurvivesOutboxReopen(t *testing.T) { t.Fatal(err) } defer reopened.Close() - if got := pendingE2EResult(t, reopened); !sameTaskResult(got, result) { + second := model.Task{ID: "task-second", LeaseID: "lease-second", Script: "# lattice-linechain-e3-v2\nsecond"} + if _, err := reopened.BeginWithProtocol(second, "linechain-e3-v2"); err != nil { + t.Fatal(err) + } + if _, err := reopened.Complete(model.TaskResult{TaskID: second.ID, LeaseID: second.LeaseID, ExitCode: 0}); err != nil { + t.Fatal(err) + } + if err := reopened.ConfirmDurability(); err != nil { + t.Fatal(err) + } + if got := pendingE2EResult(t, reopened, task.ID, task.LeaseID); !sameTaskResult(got, result) { t.Fatalf("reopened result changed: got=%+v want=%+v", got, result) } + if err := reopened.Remove(pendingE2EEntry(t, reopened, task.ID, task.LeaseID)); err != nil { + t.Fatal(err) + } + if got := pendingE2EResult(t, reopened, second.ID, second.LeaseID); got.TaskID != second.ID { + t.Fatalf("ack removed wrong durable result: %+v", got) + } +} + +func TestLinechainE2ERecoveryDoesNotCleanupBeforeDurableResult(t *testing.T) { + root := t.TempDir() + configDir, txnDir := filepath.Join(root, "conf"), filepath.Join(root, "txn") + if err := os.Mkdir(configDir, 0o700); err != nil { + t.Fatal(err) + } + sidecar := filepath.Join(root, "sidecar.json") + writeFile(t, sidecar, `{"schema":"lattice.singbox-metadata.v2","inbounds":[{"tag":"source-b","line_uuid":"bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"}]}`) + m, err := linechain.Open(txnDir) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.ConfigureLayout(configDir, sidecar); err != nil { + t.Fatal(err) + } + if err := m.ConfigureCommands("/usr/bin/true", []string{"/usr/bin/true"}, []string{"/usr/bin/true"}); err != nil { + t.Fatal(err) + } + fragment := `{"outbounds":[],"route":{"rules":[]}}` + target := "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + document := marshalE2EDocument(t, bindE2EDocument("create", "lattice-linechain-0123456789abcdef0123.json", "", &fragment, "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "source-b", nil, &target)) + script := "# lattice-linechain-e3-v2\nset -eu\n: \"${LATTICE_AGENT_BIN:?}\" \"${LATTICE_LINECHAIN_TXN_DIR:?}\"\nprintf '%s' '" + base64.StdEncoding.EncodeToString(document) + "' | base64 -d | \"$LATTICE_AGENT_BIN\" -linechain-apply\n" + task := model.Task{ID: "task-fault", LeaseID: "lease-fault", Script: script} + if err := m.Apply(context.Background(), bytes.NewReader(document), task.ID, task.LeaseID, linechainTaskScriptSHA(script)); err != nil { + t.Fatal(err) + } + result, err := m.ResolveAfterRun(context.Background(), task, model.TaskResult{TaskID: task.ID, LeaseID: task.LeaseID, ExitCode: 0}) + if err != nil { + t.Fatal(err) + } + outboxDir := filepath.Join(root, "outbox") + outbox, err := taskoutbox.Open(outboxDir) + if err != nil { + t.Fatal(err) + } + if _, err := outbox.Complete(result); err == nil { + t.Fatal("Complete without Begin unexpectedly succeeded") + } + if refs, err := m.Snapshot(); err != nil || len(refs) != 1 || !refs[0].Terminal { + t.Fatalf("transaction cleaned before durable result: refs=%+v err=%v", refs, err) + } + if _, err := outbox.BeginWithProtocol(task, "linechain-e3-v2"); err != nil { + t.Fatal(err) + } + authority, err := captureLinechainAuthority(m, outbox) + if err != nil { + t.Fatal(err) + } + if err := m.RequireRecoveredAuthorized(context.Background(), func(got model.TaskResult) error { + if !sameTaskResult(got, result) { + return fmt.Errorf("recovered result changed") + } + _, err := outbox.Complete(got) + if err != nil { + return err + } + return outbox.ConfirmDurability() + }, "node-b", authority); err != nil { + t.Fatal(err) + } + if refs, err := m.Snapshot(); err != nil || len(refs) != 0 { + t.Fatalf("transaction survived durable recovery: refs=%+v err=%v", refs, err) + } + if got := pendingE2EResult(t, outbox, task.ID, task.LeaseID); !sameTaskResult(got, result) { + t.Fatalf("durable recovered result changed: %+v", got) + } } // TestLinechainE2EInventoryHelper publishes the inventory discovered from the From 9726b430b044c4b2ddf8ab7f0e4582c6694b54fb Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:47:42 -0700 Subject: [PATCH 37/43] Prevent E2E leak detector self-matches Constraint: Cleanup scans must not match their own awk process Rejected: Literal awk command exclusion | argv formatting is platform-dependent Confidence: high Scope-risk: narrow Tested: sh -n; git diff --check Not-tested: official sing-box binary unavailable in this environment --- scripts/test-linechain-e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index a90916e..eaf05cc 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -41,7 +41,7 @@ cleanup() { process_snapshot="$e2e_root/processes" pgid_snapshot="$e2e_root/pgids" ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' - index($0, "awk -v runtime=") == 0 && (index($0, runtime) || index($0, agent_test)) { print } + (($3 == "sing-box" && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } ' >"$process_snapshot" if [ -s "$process_snapshot" ]; then echo "linechain E2E leaked process for runtime root $runtime_root" >&2 @@ -65,7 +65,7 @@ cleanup() { fi done <"$pgid_snapshot" ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' - index($0, "awk -v runtime=") == 0 && (index($0, runtime) || index($0, agent_test)) { print } + (($3 == "sing-box" && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } ' >"$process_snapshot" if [ -s "$process_snapshot" ]; then echo "linechain E2E runtime process scan remained non-empty" >&2 From 3a461ffc4726e7ae74e5ea2e014d80cc42f5cecc Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:56:15 -0700 Subject: [PATCH 38/43] Match E3 leak scans against exact sing-box executable The process table reports absolute executable paths, so cleanup scans must compare the configured binary path rather than a basename. A synthetic regression locks the strict executable and argument matching while rejecting basename and awk self-matches. Constraint: ps reports absolute executable paths Rejected: basename matching | misses the official executable Confidence: high Scope-risk: narrow Tested: sh -n scripts/test-linechain-e2e.sh; scanner regression Not-tested: full official E2E --- scripts/test-linechain-e2e.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index eaf05cc..0df0bfc 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -18,6 +18,26 @@ runtime_root="$e2e_root/runtime" mkdir -m 700 "$runtime_root" agent_bin="$e2e_root/lattice-agent" agent_test_bin="$e2e_root/lattice-agent.test" +scanner_regression() { + matches=$(printf '%s\n' \ + "101 100 $bin run -C $runtime_root" \ + "102 100 sing-box run -C $runtime_root" \ + "103 100 $agent_test_bin --runtime-root $runtime_root" \ + "104 100 awk -v runtime=$runtime_root" | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" ' + (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } + ') + [ "$(printf '%s\n' "$matches" | grep -c .)" -eq 2 ] || { + echo "process scanner regression failed: unexpected matches" >&2 + printf '%s\n' "$matches" >&2 + exit 1 + } + printf '%s\n' "$matches" | grep -F "$bin run -C $runtime_root" >/dev/null || exit 1 + printf '%s\n' "$matches" | grep -F "$agent_test_bin --runtime-root $runtime_root" >/dev/null || exit 1 +} +if [ "${LATTICE_LINECHAIN_SCANNER_REGRESSION:-}" = 1 ]; then + scanner_regression + exit 0 +fi server_dir="${LATTICE_SERVER_E2E_DIR:-}" if [ -z "$server_dir" ]; then for candidate in ../../lattice-server/.wt/worker3-task18-server ../../lattice-server; do @@ -40,8 +60,8 @@ cleanup() { trap - EXIT HUP INT TERM process_snapshot="$e2e_root/processes" pgid_snapshot="$e2e_root/pgids" - ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' - (($3 == "sing-box" && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } + ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" ' + (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } ' >"$process_snapshot" if [ -s "$process_snapshot" ]; then echo "linechain E2E leaked process for runtime root $runtime_root" >&2 @@ -64,8 +84,8 @@ cleanup() { status=1 fi done <"$pgid_snapshot" - ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v agent_test="$agent_test_bin" ' - (($3 == "sing-box" && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } + ps -axo pid=,pgid=,command= | awk -v runtime="$runtime_root" -v singbox_bin="$bin" -v agent_test="$agent_test_bin" ' + (($3 == singbox_bin && $4 == "run" && $5 == "-C" && index($0, runtime)) || ($3 == agent_test && index($0, runtime))) { print } ' >"$process_snapshot" if [ -s "$process_snapshot" ]; then echo "linechain E2E runtime process scan remained non-empty" >&2 From 028573b3e8ec2848d7cae6cf807439150bd8376d Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 11:57:45 -0700 Subject: [PATCH 39/43] Align E2E guard with current server lifecycle head The official Task29 rerun must validate against the production identity fix at the current lifecycle head rather than the stale baseline. Constraint: canonical server lifecycle head is 6fad0eb Confidence: high Scope-risk: narrow Tested: sh -n scripts/test-linechain-e2e.sh; git diff --check Not-tested: full official E2E --- scripts/test-linechain-e2e.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 0df0bfc..fbcc9a6 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -49,7 +49,8 @@ if [ -z "$server_dir" ]; then fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } [ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } -git -C "$server_dir" diff --quiet 67dc25bc6740657449b6c206cf537b22398bc289 HEAD -- . \ +# Canonical server lifecycle head for the official Task29 rerun. +git -C "$server_dir" diff --quiet 6fad0eb HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 From 4633bf91dc947f945338f15b8d05acfa42299574 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 12:00:08 -0700 Subject: [PATCH 40/43] Repin canonical E2E to indexed identity server --- scripts/test-linechain-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index fbcc9a6..38adb93 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -50,7 +50,7 @@ fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } [ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } # Canonical server lifecycle head for the official Task29 rerun. -git -C "$server_dir" diff --quiet 6fad0eb HEAD -- . \ +git -C "$server_dir" diff --quiet eabace4 HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 From fe771ed806cbe1ed900051d4552415cabc6fb9fb Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 12:05:59 -0700 Subject: [PATCH 41/43] Repin canonical E2E to shared resolver server --- scripts/test-linechain-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 38adb93..ed5f2b7 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -50,7 +50,7 @@ fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } [ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } # Canonical server lifecycle head for the official Task29 rerun. -git -C "$server_dir" diff --quiet eabace4 HEAD -- . \ +git -C "$server_dir" diff --quiet d7ef6ce HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 From b1ea42ffb10f756de6e094de7106c64e19292939 Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 12:36:51 -0700 Subject: [PATCH 42/43] Bind lifecycle verification to stable UUID authority Advance the canonical server production-tree pin so the real sing-box lifecycle exercises the batch-scoped identity resolver while preserving the strict two-file tagged-helper exception. Constraint: The lifecycle harness rejects production drift outside the exact server pin Confidence: high Scope-risk: narrow Directive: Repin only after the corresponding server commit is clean and verified Tested: bash -n scripts/test-linechain-e2e.sh; git diff --check Not-tested: lifecycle execution is the next gate --- scripts/test-linechain-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index ed5f2b7..5227552 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -50,7 +50,7 @@ fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } [ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } # Canonical server lifecycle head for the official Task29 rerun. -git -C "$server_dir" diff --quiet d7ef6ce HEAD -- . \ +git -C "$server_dir" diff --quiet 037d1d8 HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2 From 145f3e65a63a5cb560244bb92d875d8324d2496d Mon Sep 17 00:00:00 2001 From: lr00rl Date: Thu, 13 Aug 2026 12:58:01 -0700 Subject: [PATCH 43/43] Exercise lifecycle verification with node-bound identity Advance the strict server production-tree pin so the canonical real sing-box lifecycle runs against the node-owned UUID authority repair. Constraint: The harness rejects all server production drift outside its exact pin Confidence: high Scope-risk: narrow Directive: Keep this pin synchronized with the independently reviewed server authority commit Tested: bash -n scripts/test-linechain-e2e.sh; git diff --check Not-tested: lifecycle execution is the next gate --- scripts/test-linechain-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-linechain-e2e.sh b/scripts/test-linechain-e2e.sh index 5227552..7736080 100755 --- a/scripts/test-linechain-e2e.sh +++ b/scripts/test-linechain-e2e.sh @@ -50,7 +50,7 @@ fi [ -n "$server_dir" ] && [ -f "$server_dir/go.mod" ] || { echo "LATTICE_SERVER_E2E_DIR must identify the frozen lattice-server worktree" >&2; exit 1; } [ -z "$(git -C "$server_dir" status --porcelain)" ] || { echo "LATTICE_SERVER_E2E_DIR must be clean, including untracked files" >&2; exit 1; } # Canonical server lifecycle head for the official Task29 rerun. -git -C "$server_dir" diff --quiet 037d1d8 HEAD -- . \ +git -C "$server_dir" diff --quiet 611df86 HEAD -- . \ ':(exclude)internal/server/server_linechain_lifecycle_e2e_test.go' \ ':(exclude)internal/server/server_linechain_lifecycle_net_e2e_test.go' || { echo "LATTICE_SERVER_E2E_DIR must match the frozen server plus Task21 Reality repair tree" >&2