From 192514290b9875a18481869f15b3649657237001 Mon Sep 17 00:00:00 2001 From: Jonas Heinle Date: Thu, 6 Aug 2026 11:00:21 +0200 Subject: [PATCH] shim: make container teardown timeouts configurable The shim hardcodes 30s for container shutdown, for terminate, for the DeleteExec resource cleanup and for the delete command's wait. Teardown of a process isolated container is host-side work: detaching the layer filter stack and flushing the registry hives into the scratch. It scales with how much the container wrote. We measured 117s for one OpenCV build container on Windows 11 26200 with ltsc2025 base images. When 30s expires mid-flush the container is terminated while its scratch is still being written, and every later finalize of that snapshot fails with hcsshim::ExportLayer 0x3. The damage survives fresh snapshots and host reboots. All silo processes do exit, so no in-container mitigation helps: overriding WaitToKillServiceTimeout and a full pre-exit teardown of non-essential services both lost the notification the same way. Make the limits configurable via the environment the shim inherits from containerd, named after CONTAINERD_SHIM_RUNHCS_V1_WAIT_DEBUGGER: CONTAINERD_SHIM_RUNHCS_V1_TEARDOWN_TIMEOUT hcsTask.close, delete CONTAINERD_SHIM_RUNHCS_V1_TASK_CLOSE_TIMEOUT hcsTask.DeleteExec DeleteExec waits on the channel close closes, so the second is derived from the first when it is not set. Defaults stay 30s. Also log how long a successful shutdown took, which is the number needed to size the timeout. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jonas Heinle --- cmd/containerd-shim-runhcs-v1/delete.go | 3 +- cmd/containerd-shim-runhcs-v1/task_hcs.go | 66 +++++++++++++++++- .../task_hcs_test.go | 69 +++++++++++++++++++ 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/cmd/containerd-shim-runhcs-v1/delete.go b/cmd/containerd-shim-runhcs-v1/delete.go index c1eb3375fa..d35dfcbe0a 100644 --- a/cmd/containerd-shim-runhcs-v1/delete.go +++ b/cmd/containerd-shim-runhcs-v1/delete.go @@ -90,7 +90,8 @@ The delete command will be executed in the container's bundle as its cwd. } else { ch := make(chan error, 1) go func() { ch <- sys.Wait() }() - t := time.NewTimer(time.Second * 30) + // Same host-side work as [hcsTask.close], so the same bound. + t := time.NewTimer(tearDownTimeout) select { case <-t.C: sys.Close() diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index afadc50c5e..dcbf85e9e9 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -51,6 +51,62 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +// Teardown of a process isolated container is host-side work: detaching the +// layer filter stack and flushing the container's registry hives into its +// scratch. It scales with how much the container wrote, and filesystem-heavy +// workloads have been measured needing minutes. Cutting it short can leave the +// scratch in a state the platform refuses to export. Defaults are unchanged; +// hosts that need more set these on containerd, whose environment the shim +// inherits. +const ( + // Both take a Go duration string, e.g. "45m". + tearDownTimeoutEnvVar = "CONTAINERD_SHIM_RUNHCS_V1_TEARDOWN_TIMEOUT" + taskCloseTimeoutEnvVar = "CONTAINERD_SHIM_RUNHCS_V1_TASK_CLOSE_TIMEOUT" + + defaultTearDownTimeout = 30 * time.Second + defaultTaskCloseTimeout = 30 * time.Second +) + +var tearDownTimeout, taskCloseTimeout = resolveTeardownTimeouts( + os.Getenv(tearDownTimeoutEnvVar), + os.Getenv(taskCloseTimeoutEnvVar), +) + +// resolveTeardownTimeouts returns the effective timeouts. tearDown bounds each +// of the two waits in [hcsTask.close], taskClose the wait in [hcsTask.DeleteExec]. +// +// DeleteExec waits on the channel close closes, so a taskClose below close's +// worst case of 2*tearDown would abandon a teardown that is still progressing. +// When taskClose is unset and tearDown was raised, it is derived to cover that. +// +// Empty, unparseable or non-positive values count as unset: this runs at +// package init, before logging exists, and must not stop the shim starting. +func resolveTeardownTimeouts(tearDownRaw, taskCloseRaw string) (tearDown, taskClose time.Duration) { + tearDown = defaultTearDownTimeout + if d, ok := parsePositiveDuration(tearDownRaw); ok { + tearDown = d + } + + if d, ok := parsePositiveDuration(taskCloseRaw); ok { + return tearDown, d + } + if tearDown <= defaultTearDownTimeout { + return tearDown, defaultTaskCloseTimeout + } + return tearDown, 2*tearDown + defaultTaskCloseTimeout +} + +func parsePositiveDuration(s string) (time.Duration, bool) { + if s == "" { + return 0, false + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, false + } + return d, true +} + func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (shimTask, error) { log.G(ctx).WithField("tid", req.ID).Debug("newHcsStandaloneTask") @@ -536,7 +592,7 @@ func (ht *hcsTask) DeleteExec(ctx context.Context, eid string) (int, uint32, tim // If the shim exits before resources are cleaned up, those resources // will remain locked and untracked, which leads to lingering sandboxes // and container resources like base vhdx. - const timeout = 30 * time.Second + timeout := taskCloseTimeout entry.WithField(logfields.Timeout, timeout).Trace("waiting for task to be closed") select { case <-time.After(timeout): @@ -702,8 +758,6 @@ func (ht *hcsTask) close(ctx context.Context) { // method or interface for ht.c operations that we can stub for // testing. if ht.c != nil { - const tearDownTimeout = 30 * time.Second - // Do our best attempt to tear down the container. // TODO: unify timeout select statements and use [ht.c.WaitCtx] and [context.WithTimeout] var werr error @@ -717,11 +771,17 @@ func (ht *hcsTask) close(ctx context.Context) { if err != nil { entry.WithError(err).Error("failed to shutdown container") } else { + shutdownStart := time.Now() t := time.NewTimer(tearDownTimeout) select { case <-ch: err = werr t.Stop() + // The number needed to size tearDownTimeout. + entry.WithFields(logrus.Fields{ + logfields.Timeout: tearDownTimeout, + "duration": time.Since(shutdownStart).String(), + }).Debug("container shutdown completed") if err != nil { entry.WithError(err).Error("failed to wait for container shutdown") } diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs_test.go b/cmd/containerd-shim-runhcs-v1/task_hcs_test.go index d922d8e2f3..4f8c4e330c 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs_test.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs_test.go @@ -550,3 +550,72 @@ func Test_hcsTask_updateWCOWContainerCPUAffinity_XenonNotImplemented(t *testing. t.Fatalf("expected ErrNotImplemented for hypervisor-isolated container, got %v", err) } } + +func Test_resolveTeardownTimeouts(t *testing.T) { + for _, tc := range []struct { + name string + tearDownRaw string + taskCloseRaw string + tearDown time.Duration + taskClose time.Duration + }{ + { + name: "unset keeps the historical defaults", + tearDown: 30 * time.Second, + taskClose: 30 * time.Second, + }, + { + name: "raising teardown derives a task close that covers it", + tearDownRaw: "45m", + tearDown: 45 * time.Minute, + taskClose: 2*45*time.Minute + 30*time.Second, + }, + { + name: "explicit task close wins over the derived value", + tearDownRaw: "45m", + taskCloseRaw: "100m", + tearDown: 45 * time.Minute, + taskClose: 100 * time.Minute, + }, + { + name: "task close alone is honoured", + taskCloseRaw: "5m", + tearDown: 30 * time.Second, + taskClose: 5 * time.Minute, + }, + { + name: "lowering teardown does not derive", + tearDownRaw: "10s", + tearDown: 10 * time.Second, + taskClose: 30 * time.Second, + }, + { + name: "malformed and negative values fall back", + tearDownRaw: "soon", + taskCloseRaw: "-5m", + tearDown: 30 * time.Second, + taskClose: 30 * time.Second, + }, + { + name: "zero is not a positive duration", + tearDownRaw: "0s", + tearDown: 30 * time.Second, + taskClose: 30 * time.Second, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tearDown, taskClose := resolveTeardownTimeouts(tc.tearDownRaw, tc.taskCloseRaw) + if tearDown != tc.tearDown { + t.Errorf("tearDown: expected %v, got %v", tc.tearDown, tearDown) + } + if taskClose != tc.taskClose { + t.Errorf("taskClose: expected %v, got %v", tc.taskClose, taskClose) + } + // The invariant the derivation exists to protect: DeleteExec must + // not give up while close() may still be making progress. + if tc.taskCloseRaw == "" && taskClose <= 2*tearDown && tearDown > defaultTearDownTimeout { + t.Errorf("derived taskClose %v does not cover close()'s worst case of 2*%v", taskClose, tearDown) + } + }) + } +}