diff --git a/internal/runtime/lock.go b/internal/runtime/lock.go index 15ae10c9c..749bca65c 100644 --- a/internal/runtime/lock.go +++ b/internal/runtime/lock.go @@ -101,6 +101,10 @@ func (l *FileLock) Release() error { return nil } +// lockStartTimeTolerance 吸收时钟抖动:只有当占用该 PID 的进程启动时间明显晚于 +// 锁的创建时间时,才认定这个 PID 已经被系统复用。 +const lockStartTimeTolerance = time.Second + func clearStaleLock(path string) (bool, error) { raw, err := os.ReadFile(path) if err != nil { @@ -111,11 +115,39 @@ func clearStaleLock(path string) (bool, error) { } var record lockRecord if err := json.Unmarshal(raw, &record); err != nil { - return false, err + // 锁文件被截断或损坏时同样无法恢复,必须清理,否则 daemon 会永久卡死。 + return removeLockFile(path) } - if record.PID > 0 && processAlive(record.PID) { + if lockHolderActive(record) { return false, nil } + return removeLockFile(path) +} + +// lockHolderActive 判断当初写锁的进程是否仍然是当前占用该 PID 的进程。 +// +// 仅凭 processAlive 不足以判断锁仍然有效:持有者被强杀(SIGKILL、断电、launchd +// 退出超时)后锁文件会残留,一旦该 PID 被系统复用给无关进程,processAlive 就恒 +// 为真,daemon 会永久认为自己拿不到锁,只能人工删锁文件才能恢复。 +// +// 锁的创建者其进程启动时间必然不晚于锁的创建时间;若当前占用该 PID 的进程启动 +// 得更晚,它就不可能是当初写锁的那个进程。 +func lockHolderActive(record lockRecord) bool { + if record.PID <= 0 || !processAlive(record.PID) { + return false + } + if record.CreatedAt.IsZero() { + return true + } + start, ok := processStartTime(record.PID) + if !ok { + // 取不到启动时间时保持旧行为,避免误清有效锁。 + return true + } + return !start.After(record.CreatedAt.Add(lockStartTimeTolerance)) +} + +func removeLockFile(path string) (bool, error) { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return false, err } diff --git a/internal/runtime/manager_test.go b/internal/runtime/manager_test.go index 97ca6cd90..89b3f6da5 100644 --- a/internal/runtime/manager_test.go +++ b/internal/runtime/manager_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -29,6 +30,43 @@ func TestAcquireLockClearsStaleFile(t *testing.T) { defer lock.Release() } +func TestAcquireLockClearsLockWhosePIDWasReused(t *testing.T) { + pid := os.Getpid() + start, ok := processStartTime(pid) + if !ok || start.Before(time.Now().Add(-24*time.Hour)) { + t.Skip("process start time is not usable in this environment") + } + + path := filepath.Join(t.TempDir(), "relay.lock") + // 当前进程 PID 存活,但锁记录是很久以前创建的:说明这个 PID 是在锁创建之后 + // 才被系统复用给当前进程的,原持有者早已消失。 + createdAt := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano) + raw := `{"pid":` + strconv.Itoa(pid) + `,"token":"reused-pid","createdAt":"` + createdAt + `"}` + if err := os.WriteFile(path, []byte(raw), 0o600); err != nil { + t.Fatalf("write reused-pid lock: %v", err) + } + + lock, err := AcquireLock(context.Background(), path, false) + if err != nil { + t.Fatalf("AcquireLock should clear a lock whose PID was reused: %v", err) + } + defer lock.Release() +} + +func TestAcquireLockClearsCorruptLockFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "relay.lock") + // 写锁过程中被强杀会留下截断的 JSON;这种锁无法解析,同样必须清理。 + if err := os.WriteFile(path, []byte(`{"pid":`), 0o600); err != nil { + t.Fatalf("write corrupt lock: %v", err) + } + + lock, err := AcquireLock(context.Background(), path, false) + if err != nil { + t.Fatalf("AcquireLock should clear a corrupt lock file: %v", err) + } + defer lock.Release() +} + func TestAcquireLockFailsWhileLiveOwnerHoldsIt(t *testing.T) { path := filepath.Join(t.TempDir(), "relay.lock") first, err := AcquireLock(context.Background(), path, false) diff --git a/internal/runtime/process_starttime_darwin.go b/internal/runtime/process_starttime_darwin.go new file mode 100644 index 000000000..0ea6b70fe --- /dev/null +++ b/internal/runtime/process_starttime_darwin.go @@ -0,0 +1,26 @@ +//go:build darwin + +package relayruntime + +import ( + "time" + + "golang.org/x/sys/unix" +) + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + info, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil || info == nil { + return time.Time{}, false + } + start := info.Proc.P_starttime + if start.Sec == 0 && start.Usec == 0 { + return time.Time{}, false + } + return time.Unix(start.Sec, int64(start.Usec)*1000), true +} diff --git a/internal/runtime/process_starttime_linux.go b/internal/runtime/process_starttime_linux.go new file mode 100644 index 000000000..759573e26 --- /dev/null +++ b/internal/runtime/process_starttime_linux.go @@ -0,0 +1,67 @@ +//go:build linux + +package relayruntime + +import ( + "os" + "strconv" + "strings" + "time" +) + +// linuxClockTicksPerSecond 是 /proc//stat 中 starttime 的计数频率(USER_HZ)。 +// Linux 上该值固定为 100,且 Go 在不引入 cgo 的前提下无法查询 sysconf。 +const linuxClockTicksPerSecond = 100 + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") + if err != nil { + return time.Time{}, false + } + // comm 字段(字段 2)可能包含空格和括号,因此从最后一个 ')' 之后开始切分。 + text := string(raw) + cut := strings.LastIndex(text, ")") + if cut < 0 || cut+2 > len(text) { + return time.Time{}, false + } + fields := strings.Fields(text[cut+2:]) + // ')' 之后第一个字段是 state(字段 3),starttime 是字段 22。 + const startTimeField = 22 - 3 + if len(fields) <= startTimeField { + return time.Time{}, false + } + ticks, err := strconv.ParseInt(fields[startTimeField], 10, 64) + if err != nil { + return time.Time{}, false + } + boot, ok := linuxBootTime() + if !ok { + return time.Time{}, false + } + return boot.Add(time.Duration(ticks) * time.Second / linuxClockTicksPerSecond), true +} + +// linuxBootTime 读取系统启动时刻(/proc/stat 的 btime 行)。 +func linuxBootTime() (time.Time, bool) { + raw, err := os.ReadFile("/proc/stat") + if err != nil { + return time.Time{}, false + } + for _, line := range strings.Split(string(raw), "\n") { + value, found := strings.CutPrefix(line, "btime ") + if !found { + continue + } + seconds, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(seconds, 0), true + } + return time.Time{}, false +} diff --git a/internal/runtime/process_starttime_other.go b/internal/runtime/process_starttime_other.go new file mode 100644 index 000000000..f80cbd9e3 --- /dev/null +++ b/internal/runtime/process_starttime_other.go @@ -0,0 +1,10 @@ +//go:build !darwin && !linux && !windows + +package relayruntime + +import "time" + +// processStartTime 在缺少实现的平台上不可用;此时锁判定退化为只检查 PID 是否存活。 +func processStartTime(int) (time.Time, bool) { + return time.Time{}, false +} diff --git a/internal/runtime/process_starttime_windows.go b/internal/runtime/process_starttime_windows.go new file mode 100644 index 000000000..0ed393758 --- /dev/null +++ b/internal/runtime/process_starttime_windows.go @@ -0,0 +1,31 @@ +//go:build windows + +package relayruntime + +import ( + "time" + + "golang.org/x/sys/windows" +) + +// processStartTime 返回 pid 对应进程的启动时间。 +// 平台不支持或进程不存在时返回 ok=false。 +func processStartTime(pid int) (time.Time, bool) { + if pid <= 0 { + return time.Time{}, false + } + handle, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return time.Time{}, false + } + defer windows.CloseHandle(handle) + var creation, exit, kernel, user windows.Filetime + if err := windows.GetProcessTimes(handle, &creation, &exit, &kernel, &user); err != nil { + return time.Time{}, false + } + if creation == (windows.Filetime{}) { + return time.Time{}, false + } + // Filetime.Nanoseconds 已经换算为 Unix 纪元起的纳秒。 + return time.Unix(0, creation.Nanoseconds()), true +}