From 48442ecb09a8d88e02001f2dc4d87553a6346c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pei=20Qing=20=E5=8D=BF=E5=9F=B9?= Date: Wed, 16 Sep 2026 18:19:33 +0800 Subject: [PATCH] fix(runtime): treat a reused PID as a stale lock clearStaleLock only checked processAlive on the recorded PID. After the lock holder is killed without a chance to clean up (SIGKILL, power loss, launchd exit timeout), the lock file stays behind. As soon as that PID is recycled by any unrelated process, processAlive stays true forever and the daemon can never acquire the lock again: it exits with "lock held by another process" on every start, which launchd's KeepAlive amplifies into a crash loop until the lock file is deleted by hand. The same function also returned an error instead of clearing the file when the lock JSON could not be parsed, so a lock truncated mid-write had the same permanent effect. Compare the holder's process start time with the lock creation time: the process that wrote the lock cannot have started after the lock was created, so a later start time means the PID was reused. Keep the previous behaviour when the start time is unavailable, and clear unparseable lock files. Start time lookups are implemented for darwin, linux and windows. Fixes #913 --- internal/runtime/lock.go | 36 +++++++++- internal/runtime/manager_test.go | 38 +++++++++++ internal/runtime/process_starttime_darwin.go | 26 +++++++ internal/runtime/process_starttime_linux.go | 67 +++++++++++++++++++ internal/runtime/process_starttime_other.go | 10 +++ internal/runtime/process_starttime_windows.go | 31 +++++++++ 6 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 internal/runtime/process_starttime_darwin.go create mode 100644 internal/runtime/process_starttime_linux.go create mode 100644 internal/runtime/process_starttime_other.go create mode 100644 internal/runtime/process_starttime_windows.go diff --git a/internal/runtime/lock.go b/internal/runtime/lock.go index 15ae10c9c..749bca65c 100644 --- a/internal/runtime/lock.go +++ b/internal/runtime/lock.go @@ -101,6 +101,10 @@ func (l *FileLock) Release() error { return nil } +// lockStartTimeTolerance 吸收时钟抖动：只有当占用该 PID 的进程启动时间明显晚于 +// 锁的创建时间时，才认定这个 PID 已经被系统复用。 +const lockStartTimeTolerance = time.Second + func clearStaleLock(path string) (bool, error) { raw, err := os.ReadFile(path) if err != nil { @@ -111,11 +115,39 @@ func clearStaleLock(path string) (bool, error) { } var record lockRecord if err := json.Unmarshal(raw, &record); err != nil { - return false, err + // 锁文件被截断或损坏时同样无法恢复，必须清理，否则 daemon 会永久卡死。 + return removeLockFile(path) } - if record.PID > 0 && processAlive(record.PID) { + if lockHolderActive(record) { return false, nil } + return removeLockFile(path) +} + +// lockHolderActive 判断当初写锁的进程是否仍然是当前占用该 PID 的进程。 +// +// 仅凭 processAlive 不足以判断锁仍然有效：持有者被强杀（SIGKILL、断电、launchd +// 退出超时）后锁文件会残留，一旦该 PID 被系统复用给无关进程，processAlive 就恒 +// 为真，daemon 会永久认为自己拿不到锁，只能人工删锁文件才能恢复。 +// +// 锁的创建者其进程启动时间必然不晚于锁的创建时间；若当前占用该 PID 的进程启动 +// 得更晚，它就不可能是当初写锁的那个进程。 +func lockHolderActive(record lockRecord) bool { + if record.PID <= 0 || !processAlive(record.PID) { + return false + } + if record.CreatedAt.IsZero() { + return true + } + start, ok := processStartTime(record.PID) + if !ok { + // 取不到启动时间时保持旧行为，避免误清有效锁。 + return true + } + return !start.After(record.CreatedAt.Add(lockStartTimeTolerance)) +} + +func removeLockFile(path string) (bool, error) { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return false, err } diff --git a/internal/runtime/manager_test.go b/internal/runtime/manager_test.go index 97ca6cd90..89b3f6da5 100644 --- a/internal/runtime/manager_test.go +++ b/internal/runtime/manager_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -29,6 +30,43 @@ func TestAcquireLockClearsStaleFile(t *testing.T) { defer lock.Release() } +func TestAcquireLockClearsLockWhosePIDWasReused(t *testing.T) { + pid := os.Getpid() + start, ok := processStartTime(pid) + if !ok || start.Before(time.Now().Add(-24*time.Hour)) { + t.Skip("process start time is not usable in this environment") + } + + path := filepath.Join(t.TempDir(), "relay.lock") + // 当前进程 PID 存活，但锁记录是很久以前创建的：说明这个 PID 是在锁创建之后 + // 才被系统复用给当前进程的，原持有者早已消失。 + createdAt := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano) + raw := `{"pid":` + strconv.Itoa(pid) + `,"token":"reused-pid","createdAt":"` + createdAt + `"}` + if err := os.WriteFile(path, []byte(raw), 0o600); err != nil { + t.Fatalf("write reused-pid lock: %v", err) + } + + lock, err := AcquireLock(context.Background(), path, false) + if err != nil { + t.Fatalf("AcquireLock should clear a lock whose PID was reused: %v", err) + } + defer lock.Release() +} + +func TestAcquireLockClearsCorruptLockFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "relay.lock") + // 写锁过程中被强杀会留下截断的 JSON；这种锁无法解析，同样必须清理。 + if err := os.WriteFile(path, []byte(`{"pid":`), 0o600); err != nil { + t.Fatalf("write corrupt lock: %v", err) + } + + lock, err := AcquireLock(context.Background(), path, false) + if err != nil { + t.Fatalf("AcquireLock should clear a corrupt lock file: %v", err) + } + defer lock.Release() +} + func TestAcquireLockFailsWhileLiveOwnerHoldsIt(t *testing.T) { path := filepath.Join(t.TempDir(), "relay.lock") first, err := AcquireLock(context.Background(), path, false) diff --git a/internal/runtime/process_starttime_darwin.go b/internal/runtime/process_starttime_darwin.go new file mode 100644 index 000000000..0ea6b70fe --- /dev/null +++ b/internal/runtime/process_starttime_darwin.go @@ -0,0 +1,26 @@ +//go:build darwin + +package relayruntime + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + info, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil || info == nil { + return time.Time{}, false + } + start := info.Proc.P_starttime + if start.Sec == 0 && start.Usec == 0 { + return time.Time{}, false + } + return time.Unix(start.Sec, int64(start.Usec)*1000), true +} diff --git a/internal/runtime/process_starttime_linux.go b/internal/runtime/process_starttime_linux.go new file mode 100644 index 000000000..759573e26 --- /dev/null +++ b/internal/runtime/process_starttime_linux.go @@ -0,0 +1,67 @@ +//go:build linux + +package relayruntime + +import ( + "os" + "strconv" + "strings" + "time" +) + +// linuxClockTicksPerSecond 是 /proc//stat 中 starttime 的计数频率（USER_HZ）。 +// Linux 上该值固定为 100，且 Go 在不引入 cgo 的前提下无法查询 sysconf。 +const linuxClockTicksPerSecond = 100 + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") + if err != nil { + return time.Time{}, false + } + // comm 字段（字段 2）可能包含空格和括号，因此从最后一个 ')' 之后开始切分。 + text := string(raw) + cut := strings.LastIndex(text, ")") + if cut < 0 || cut+2 > len(text) { + return time.Time{}, false + } + fields := strings.Fields(text[cut+2:]) + // ')' 之后第一个字段是 state（字段 3），starttime 是字段 22。 + const startTimeField = 22 - 3 + if len(fields) <= startTimeField { + return time.Time{}, false + } + ticks, err := strconv.ParseInt(fields[startTimeField], 10, 64) + if err != nil { + return time.Time{}, false + } + boot, ok := linuxBootTime() + if !ok { + return time.Time{}, false + } + return boot.Add(time.Duration(ticks) * time.Second / linuxClockTicksPerSecond), true +} + +// linuxBootTime 读取系统启动时刻（/proc/stat 的 btime 行）。 +func linuxBootTime() (time.Time, bool) { + raw, err := os.ReadFile("/proc/stat") + if err != nil { + return time.Time{}, false + } + for _, line := range strings.Split(string(raw), "\n") { + value, found := strings.CutPrefix(line, "btime ") + if !found { + continue + } + seconds, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(seconds, 0), true + } + return time.Time{}, false +} diff --git a/internal/runtime/process_starttime_other.go b/internal/runtime/process_starttime_other.go new file mode 100644 index 000000000..f80cbd9e3 --- /dev/null +++ b/internal/runtime/process_starttime_other.go @@ -0,0 +1,10 @@ +//go:build !darwin && !linux && !windows + +package relayruntime + +import "time" + +// processStartTime 在缺少实现的平台上不可用；此时锁判定退化为只检查 PID 是否存活。 +func processStartTime(int) (time.Time, bool) { + return time.Time{}, false +} diff --git a/internal/runtime/process_starttime_windows.go b/internal/runtime/process_starttime_windows.go new file mode 100644 index 000000000..0ed393758 --- /dev/null +++ b/internal/runtime/process_starttime_windows.go @@ -0,0 +1,31 @@ +//go:build windows + +package relayruntime + +import ( + "time" + + "golang.org/x/sys/windows" +) + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return time.Time{}, false + } + defer windows.CloseHandle(handle) + var creation, exit, kernel, user windows.Filetime + if err := windows.GetProcessTimes(handle, &creation, &exit, &kernel, &user); err != nil { + return time.Time{}, false + } + if creation == (windows.Filetime{}) { + return time.Time{}, false + } + // Filetime.Nanoseconds 已经换算为 Unix 纪元起的纳秒。 + return time.Unix(0, creation.Nanoseconds()), true +}