diff --git a/go.mod b/go.mod index 5aece2fa7..efc5b4632 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/visualfc/gid v0.3.3 golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd golang.org/x/mod v0.32.0 + golang.org/x/sys v0.41.0 golang.org/x/tools v0.41.0 ) diff --git a/go.sum b/go.sum index e151773a7..a4bad6ada 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/processsupervisor/run_unix.go b/internal/processsupervisor/run_unix.go new file mode 100644 index 000000000..fa380b561 --- /dev/null +++ b/internal/processsupervisor/run_unix.go @@ -0,0 +1,199 @@ +//go:build !windows + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processsupervisor + +import ( + "context" + "errors" + "os" + "os/exec" + "syscall" + "time" +) + +const ( + unixShutdownGrace = 200 * time.Millisecond + unixGroupPoll = 10 * time.Millisecond +) + +func run(ctx context.Context, cmd *exec.Cmd) (Status, error) { + // Keep any caller-provided attributes, but make the child the leader of a + // private process group. Forwarding to -pid then reaches the Engine and + // descendants without ever signalling this wrapper's process group. + attr := cmd.SysProcAttr + if attr == nil { + attr = &syscall.SysProcAttr{} + } else { + copy := *attr + attr = © + } + attr.Setpgid = true + cmd.SysProcAttr = attr + + // CommandContext's default Cancel kills only the leader. Replace it with a + // request that this supervisor turns into TERM -> grace -> KILL for the + // whole group. A nil Cancel identifies a plain exec.Command and must remain + // nil because os/exec rejects Cancel on commands without an internal context. + cancelRequests := make(chan struct{}, 1) + if cmd.Cancel != nil { + cmd.Cancel = func() error { + select { + case cancelRequests <- struct{}{}: + default: + } + return nil + } + } + // Bound pipe-copy goroutines if a descendant inherits stdout/stderr after + // the leader exits. Group termination is scheduled sooner than this direct- + // leader os/exec fallback, so supervisor semantics remain authoritative. + cmd.WaitDelay = 2 * unixShutdownGrace + + if err := cmd.Start(); err != nil { + return Status{}, err + } + pgid := cmd.Process.Pid + + type waitResult struct { + state *os.ProcessState + err error + } + wait := make(chan waitResult, 1) + go func() { + err := cmd.Wait() + wait <- waitResult{state: cmd.ProcessState, err: err} + }() + + ctxDone := ctx.Done() + var shutdownDeadline time.Time + var killTimer *time.Timer + var killTimerC <-chan time.Time + var shutdownErr error + var cancellationObserved bool + beginShutdown := func() { + if !shutdownDeadline.IsZero() { + return + } + shutdownDeadline = time.Now().Add(unixShutdownGrace) + shutdownSignal := syscall.SIGTERM + if signal, ok := signalFromContext(ctx); ok { + if unixSignal, ok := signal.(syscall.Signal); ok { + shutdownSignal = unixSignal + } + } + if err := signalProcessGroup(pgid, shutdownSignal); err != nil { + shutdownErr = errors.Join(shutdownErr, err) + } + killTimer = time.NewTimer(unixShutdownGrace) + killTimerC = killTimer.C + } + + for { + select { + case result := <-wait: + if killTimer != nil { + killTimer.Stop() + } + // Waiting for the leader is not enough: descendants remain in its + // process group. Give them the same short graceful shutdown window, + // then kill any remainder before returning to caller cleanup. + cleanupErr := cleanupProcessGroup(pgid, shutdownDeadline) + if err := errors.Join(shutdownErr, cleanupErr); err != nil { + return Status{}, err + } + if result.err == nil { + if err := cancellationError(ctx, cancellationObserved); err != nil { + return Status{}, err + } + return Status{}, nil + } + var exitError *exec.ExitError + if !errors.As(result.err, &exitError) || result.state == nil { + return Status{}, result.err + } + return statusFromProcessState(result.state), nil + case <-cancelRequests: + cancellationObserved = true + beginShutdown() + case <-ctxDone: + ctxDone = nil + cancellationObserved = true + beginShutdown() + case <-killTimerC: + killTimerC = nil + if err := signalProcessGroup(pgid, syscall.SIGKILL); err != nil { + shutdownErr = errors.Join(shutdownErr, err) + } + } + } +} + +func cleanupProcessGroup(pgid int, deadline time.Time) error { + alive, err := processGroupAlive(pgid) + if err != nil || !alive { + return err + } + if deadline.IsZero() { + if err := signalProcessGroup(pgid, syscall.SIGTERM); err != nil { + return err + } + deadline = time.Now().Add(unixShutdownGrace) + } + for time.Now().Before(deadline) { + alive, err = processGroupAlive(pgid) + if err != nil || !alive { + return err + } + remaining := time.Until(deadline) + if remaining > unixGroupPoll { + remaining = unixGroupPoll + } + time.Sleep(remaining) + } + return signalProcessGroup(pgid, syscall.SIGKILL) +} + +func processGroupAlive(pgid int) (bool, error) { + err := syscall.Kill(-pgid, 0) + switch { + case err == nil, errors.Is(err, syscall.EPERM): + return true, nil + case errors.Is(err, syscall.ESRCH): + return false, nil + default: + return false, err + } +} + +func signalProcessGroup(pgid int, sig syscall.Signal) error { + if err := syscall.Kill(-pgid, sig); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} + +func statusFromProcessState(state *os.ProcessState) Status { + if state == nil { + return Status{Code: 1} + } + if wait, ok := state.Sys().(syscall.WaitStatus); ok && wait.Signaled() { + return Status{Signal: int(wait.Signal())} + } + return Status{Code: state.ExitCode()} +} diff --git a/internal/processsupervisor/run_windows.go b/internal/processsupervisor/run_windows.go new file mode 100644 index 000000000..fb04546b6 --- /dev/null +++ b/internal/processsupervisor/run_windows.go @@ -0,0 +1,212 @@ +//go:build windows + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processsupervisor + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +const windowsWaitDelay = 400 * time.Millisecond + +// Windows has no POSIX process-group signal semantics. The child is created +// suspended, assigned to a KILL_ON_JOB_CLOSE Job Object, and only then resumed. +// This closes the previous assignment race in which the child could create an +// untracked descendant before job ownership was established. +func run(ctx context.Context, cmd *exec.Cmd) (Status, error) { + attr := cmd.SysProcAttr + if attr == nil { + attr = &syscall.SysProcAttr{} + } else { + copy := *attr + attr = © + } + attr.CreationFlags |= windows.CREATE_NEW_PROCESS_GROUP | windows.CREATE_SUSPENDED + cmd.SysProcAttr = attr + + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return Status{}, fmt.Errorf("processsupervisor: create Windows job: %w", err) + } + jobOpen := true + defer func() { + if jobOpen { + _ = windows.CloseHandle(job) + } + }() + limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{} + limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil { + return Status{}, fmt.Errorf("processsupervisor: configure Windows job: %w", err) + } + + // CommandContext's default Cancel targets only the leader. Queue a whole-job + // termination request instead. Leave nil Cancel untouched for exec.Command, + // which has no internal context and rejects a non-nil Cancel during Start. + cancelRequests := make(chan struct{}, 1) + if cmd.Cancel != nil { + cmd.Cancel = func() error { + select { + case cancelRequests <- struct{}{}: + default: + } + return nil + } + } + cmd.WaitDelay = windowsWaitDelay + + if err := cmd.Start(); err != nil { + return Status{}, err + } + abort := func(cause error) (Status, error) { + // Every failure after suspended creation must reap the child. Terminate + // the job if assignment happened, directly kill as the fail-closed + // fallback, then call Cmd.Wait to release os/exec resources. + _ = windows.TerminateJobObject(job, 1) + _ = cmd.Process.Kill() + _ = cmd.Wait() + return Status{}, cause + } + + processHandle, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(cmd.Process.Pid)) + if err != nil { + return abort(fmt.Errorf("processsupervisor: open suspended Windows child: %w", err)) + } + assignErr := windows.AssignProcessToJobObject(job, processHandle) + closeProcessErr := windows.CloseHandle(processHandle) + if assignErr != nil { + return abort(fmt.Errorf("processsupervisor: assign suspended child to Windows job: %w", assignErr)) + } + if closeProcessErr != nil { + return abort(fmt.Errorf("processsupervisor: close Windows child handle: %w", closeProcessErr)) + } + + thread, err := openSuspendedProcessThread(uint32(cmd.Process.Pid)) + if err != nil { + return abort(fmt.Errorf("processsupervisor: open suspended Windows child thread: %w", err)) + } + if ctx.Err() != nil { + _ = windows.CloseHandle(thread) + return abort(cancellationError(ctx, true)) + } + resumeErr := resumeWindowsThread(thread) + closeThreadErr := windows.CloseHandle(thread) + if resumeErr != nil { + return abort(fmt.Errorf("processsupervisor: resume Windows child: %w", resumeErr)) + } + if closeThreadErr != nil { + return abort(fmt.Errorf("processsupervisor: close Windows child thread handle: %w", closeThreadErr)) + } + + type waitResult struct { + state *os.ProcessState + err error + } + wait := make(chan waitResult, 1) + go func() { + err := cmd.Wait() + wait <- waitResult{state: cmd.ProcessState, err: err} + }() + ctxDone := ctx.Done() + var cancellationObserved bool + for { + select { + case result := <-wait: + // Closing a KILL_ON_JOB_CLOSE job terminates descendants that outlive + // the leader. Do it before returning so caller cleanup cannot race a + // surviving Engine child tree. + if err := windows.CloseHandle(job); err != nil { + return Status{}, fmt.Errorf("processsupervisor: close Windows job: %w", err) + } + jobOpen = false + if result.err == nil { + if err := cancellationError(ctx, cancellationObserved); err != nil { + return Status{}, err + } + return Status{}, nil + } + var exitError *exec.ExitError + if !errors.As(result.err, &exitError) || result.state == nil { + return Status{}, result.err + } + return statusFromProcessState(result.state), nil + case <-cancelRequests: + cancellationObserved = true + _ = windows.TerminateJobObject(job, 1) + case <-ctxDone: + ctxDone = nil + cancellationObserved = true + _ = windows.TerminateJobObject(job, 1) + } + } +} + +func openSuspendedProcessThread(pid uint32) (windows.Handle, error) { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return 0, err + } + defer windows.CloseHandle(snapshot) + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + if err := windows.Thread32First(snapshot, &entry); err != nil { + return 0, err + } + for { + if entry.OwnerProcessID == pid { + return windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + } + entry.Size = uint32(unsafe.Sizeof(entry)) + if err := windows.Thread32Next(snapshot, &entry); err != nil { + if errors.Is(err, syscall.ERROR_NO_MORE_FILES) { + return 0, fmt.Errorf("no thread found for process %d", pid) + } + return 0, err + } + } +} + +func resumeWindowsThread(thread windows.Handle) error { + for { + previous, err := windows.ResumeThread(thread) + if err != nil { + return err + } + if previous == 0 { + return errors.New("processsupervisor: Windows child thread was not suspended") + } + if previous == 1 { + return nil + } + } +} + +func statusFromProcessState(state *os.ProcessState) Status { + if state == nil { + return Status{Code: 1} + } + return Status{Code: state.ExitCode()} +} diff --git a/internal/processsupervisor/supervisor.go b/internal/processsupervisor/supervisor.go new file mode 100644 index 000000000..934f88e4c --- /dev/null +++ b/internal/processsupervisor/supervisor.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package processsupervisor runs one already-prepared child command while +// keeping process lifetime and process-status handling at a library boundary. +// It never changes the caller's cwd/environment and never terminates the +// calling process. +package processsupervisor + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" +) + +// Status is the terminal status of a child. A non-zero Signal means the child +// was terminated by that signal; Code is used only for normal termination. +// Windows cannot preserve POSIX signal identity and reports only Code. +type Status struct { + Code int + Signal int +} + +// SignalCause carries a signal intercepted by the outer command boundary +// through context cancellation. On Unix, Run consumes this cause when stopping +// a child group instead of subscribing to process-wide signals itself. +type SignalCause struct { + Signal os.Signal +} + +func (c *SignalCause) Error() string { + if c == nil || c.Signal == nil { + return "processsupervisor: canceled by signal" + } + return fmt.Sprintf("processsupervisor: canceled by signal %s", c.Signal) +} + +func signalFromContext(ctx context.Context) (os.Signal, bool) { + var cause *SignalCause + if !errors.As(context.Cause(ctx), &cause) || cause == nil || cause.Signal == nil { + return nil, false + } + return cause.Signal, true +} + +func cancellationError(ctx context.Context, observed bool) error { + if !observed && ctx.Err() == nil { + return nil + } + if cause := context.Cause(ctx); cause != nil { + return cause + } + return context.Canceled +} + +// Success reports whether the child exited normally with status zero. +func (s Status) Success() bool { return s.Signal == 0 && s.Code == 0 } + +// Run starts and waits for cmd. Preparation/start failures and wait failures +// are returned as errors. A child exit (including a non-zero exit code or a +// signal termination) is represented by Status and is not an error. On Unix, +// host applications own signal subscription and may cancel ctx with +// SignalCause. +func Run(ctx context.Context, cmd *exec.Cmd) (Status, error) { + if ctx == nil { + return Status{}, errors.New("processsupervisor: nil context") + } + if cmd == nil { + return Status{}, errors.New("processsupervisor: nil command") + } + if cmd.Process != nil { + return Status{}, errors.New("processsupervisor: command has already started") + } + return run(ctx, cmd) +} diff --git a/internal/processsupervisor/supervisor_test.go b/internal/processsupervisor/supervisor_test.go new file mode 100644 index 000000000..02927f744 --- /dev/null +++ b/internal/processsupervisor/supervisor_test.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processsupervisor + +import ( + "context" + "errors" + "testing" +) + +func TestCancellationError(t *testing.T) { + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + + cause := errors.New("custom cancellation cause") + causedCtx, cancelCause := context.WithCancelCause(context.Background()) + cancelCause(cause) + + tests := []struct { + name string + ctx context.Context + observed bool + want error + }{ + {name: "not canceled", ctx: context.Background()}, + {name: "observed command cancellation", ctx: context.Background(), observed: true, want: context.Canceled}, + {name: "context canceled before observation", ctx: canceledCtx, want: context.Canceled}, + {name: "context cancellation cause", ctx: causedCtx, want: cause}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cancellationError(tt.ctx, tt.observed) + if !errors.Is(got, tt.want) { + t.Fatalf("cancellationError() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/processsupervisor/supervisor_unix_test.go b/internal/processsupervisor/supervisor_unix_test.go new file mode 100644 index 000000000..a007284a5 --- /dev/null +++ b/internal/processsupervisor/supervisor_unix_test.go @@ -0,0 +1,274 @@ +//go:build !windows + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processsupervisor + +import ( + "context" + "errors" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestRunSeparatesPreparationErrorFromChildExit(t *testing.T) { + status, err := Run(context.Background(), exec.Command(filepath.Join(t.TempDir(), "missing"))) + if err == nil || status != (Status{}) { + t.Fatalf("preparation result = (%+v, %v), want zero status and error", status, err) + } + + command := exec.Command(os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), "SPX_PROCESS_SUPERVISOR_HELPER=exit1") + status, err = Run(context.Background(), command) + if err != nil || status != (Status{Code: 1}) { + t.Fatalf("child exit result = (%+v, %v), want code 1 and nil error", status, err) + } +} + +func TestRunForwardsSignalCauseToChildGroup(t *testing.T) { + marker := filepath.Join(t.TempDir(), "started") + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), "SPX_PROCESS_SUPERVISOR_HELPER=wait", "SPX_PROCESS_SUPERVISOR_MARKER="+marker) + result := make(chan struct { + status Status + err error + }, 1) + go func() { + status, err := Run(ctx, command) + result <- struct { + status Status + err error + }{status: status, err: err} + }() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(marker); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("child did not start") + } + time.Sleep(10 * time.Millisecond) + } + cancel(&SignalCause{Signal: syscall.SIGINT}) + select { + case result := <-result: + if result.err != nil || result.status != (Status{Signal: int(syscall.SIGINT)}) { + t.Fatalf("forwarded signal result = (%+v, %v), want SIGINT and nil error", result.status, result.err) + } + case <-time.After(5 * time.Second): + t.Fatal("supervisor did not finish after forwarded signal") + } +} + +func TestRunReturnsCancellationAfterGracefulExit(t *testing.T) { + marker := filepath.Join(t.TempDir(), "started") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), + "SPX_PROCESS_SUPERVISOR_HELPER=graceful-exit", + "SPX_PROCESS_SUPERVISOR_MARKER="+marker, + ) + result := make(chan struct { + status Status + err error + }, 1) + go func() { + status, err := Run(ctx, command) + result <- struct { + status Status + err error + }{status: status, err: err} + }() + waitForFile(t, marker) + cancel() + select { + case result := <-result: + if result.err == nil || !errors.Is(result.err, context.Canceled) || result.status != (Status{}) { + t.Fatalf("graceful cancellation result = (%+v, %v), want context.Canceled and zero status", result.status, result.err) + } + case <-time.After(5 * time.Second): + t.Fatal("supervisor did not finish after graceful cancellation") + } +} + +func TestRunCleansGrandchildAfterLeaderExit(t *testing.T) { + marker := filepath.Join(t.TempDir(), "grandchild") + command := exec.Command(os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), + "SPX_PROCESS_SUPERVISOR_HELPER=spawn-grandchild", + "SPX_PROCESS_SUPERVISOR_MARKER="+marker, + ) + status, err := Run(context.Background(), command) + if err != nil || status != (Status{}) { + t.Fatalf("leader result = (%+v, %v), want successful status", status, err) + } + pid := readHelperPID(t, marker) + waitForProcessGone(t, pid) +} + +func TestRunContextEscalatesIgnoredTerm(t *testing.T) { + marker := filepath.Join(t.TempDir(), "leader") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), + "SPX_PROCESS_SUPERVISOR_HELPER=ignore-term", + "SPX_PROCESS_SUPERVISOR_MARKER="+marker, + ) + result := make(chan struct { + status Status + err error + }, 1) + go func() { + status, err := Run(ctx, command) + result <- struct { + status Status + err error + }{status: status, err: err} + }() + waitForFile(t, marker) + cancel() + select { + case result := <-result: + if result.err != nil || result.status != (Status{Signal: int(syscall.SIGKILL)}) { + t.Fatalf("canceled result = (%+v, %v), want SIGKILL after TERM grace", result.status, result.err) + } + case <-time.After(5 * time.Second): + t.Fatal("supervisor did not escalate ignored TERM") + } + waitForProcessGone(t, readHelperPID(t, marker)) +} + +func TestProcessSupervisorHelper(t *testing.T) { + if os.Getenv("SPX_PROCESS_SUPERVISOR_GRANDCHILD") == "ignore-term" { + runIgnoringTermHelper() + return + } + switch os.Getenv("SPX_PROCESS_SUPERVISOR_HELPER") { + case "exit1": + os.Exit(1) + case "wait": + if marker := os.Getenv("SPX_PROCESS_SUPERVISOR_MARKER"); marker != "" { + _ = os.WriteFile(marker, []byte("started"), 0o600) + } + for { + time.Sleep(time.Second) + } + case "graceful-exit": + runGracefulExitHelper() + case "ignore-term": + runIgnoringTermHelper() + case "spawn-grandchild": + command := exec.Command(os.Args[0], "-test.run=^TestProcessSupervisorHelper$", "--") + command.Env = append(os.Environ(), "SPX_PROCESS_SUPERVISOR_GRANDCHILD=ignore-term") + if err := command.Start(); err != nil { + os.Exit(71) + } + if !waitForFilePath(os.Getenv("SPX_PROCESS_SUPERVISOR_MARKER"), 5*time.Second) { + os.Exit(72) + } + os.Exit(0) + } +} + +func runGracefulExitHelper() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGTERM) + marker := os.Getenv("SPX_PROCESS_SUPERVISOR_MARKER") + if marker == "" || os.WriteFile(marker, []byte("started"), 0o600) != nil { + os.Exit(74) + } + <-signals + // Bypass os.Exit's race-detector finalization. Under -race, that finalizer + // can exceed the supervisor's intentionally short grace period and turn a + // graceful helper into a synthetic SIGKILL. + syscall.Exit(0) +} + +func runIgnoringTermHelper() { + signal.Ignore(syscall.SIGTERM) + marker := os.Getenv("SPX_PROCESS_SUPERVISOR_MARKER") + if marker == "" || os.WriteFile(marker, []byte(strconv.Itoa(os.Getpid())), 0o600) != nil { + os.Exit(73) + } + for { + time.Sleep(time.Second) + } +} + +func readHelperPID(t *testing.T, marker string) int { + t.Helper() + waitForFile(t, marker) + data, err := os.ReadFile(marker) + if err != nil { + t.Fatal(err) + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || pid <= 0 { + t.Fatalf("helper pid marker = %q: %v", data, err) + } + return pid +} + +func waitForFile(t *testing.T, path string) { + t.Helper() + if !waitForFilePath(path, 5*time.Second) { + t.Fatalf("file %q was not created", path) + } +} + +func waitForFilePath(path string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + if _, err := os.Stat(path); err == nil { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(10 * time.Millisecond) + } +} + +func waitForProcessGone(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + err := syscall.Kill(pid, 0) + if errors.Is(err, syscall.ESRCH) { + return + } + if err != nil && !errors.Is(err, syscall.EPERM) { + t.Fatalf("inspect process %d: %v", pid, err) + } + if time.Now().After(deadline) { + t.Fatalf("process %d survived supervisor cleanup", pid) + } + time.Sleep(10 * time.Millisecond) + } +}