shim: make container teardown timeouts configurable

cmd/containerd-shim-runhcs-v1/delete.go

@@ -90,7 +90,8 @@ The delete command will be executed in the container's bundle as its cwd.
 			} else {
 				ch := make(chan error, 1)
 				go func() { ch <- sys.Wait() }()
-				t := time.NewTimer(time.Second * 30)
+				// Same host-side work as [hcsTask.close], so the same bound.
+				t := time.NewTimer(tearDownTimeout)
 				select {
 				case <-t.C:
 					sys.Close()

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -51,6 +51,62 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
+// Teardown of a process isolated container is host-side work: detaching the
+// layer filter stack and flushing the container's registry hives into its
+// scratch. It scales with how much the container wrote, and filesystem-heavy
+// workloads have been measured needing minutes. Cutting it short can leave the
+// scratch in a state the platform refuses to export. Defaults are unchanged;
+// hosts that need more set these on containerd, whose environment the shim
+// inherits.
+const (
+	// Both take a Go duration string, e.g. "45m".
+	tearDownTimeoutEnvVar  = "CONTAINERD_SHIM_RUNHCS_V1_TEARDOWN_TIMEOUT"
+	taskCloseTimeoutEnvVar = "CONTAINERD_SHIM_RUNHCS_V1_TASK_CLOSE_TIMEOUT"
+
+	defaultTearDownTimeout  = 30 * time.Second
+	defaultTaskCloseTimeout = 30 * time.Second
+)
+
+var tearDownTimeout, taskCloseTimeout = resolveTeardownTimeouts(
+	os.Getenv(tearDownTimeoutEnvVar),
+	os.Getenv(taskCloseTimeoutEnvVar),
+)
+
+// resolveTeardownTimeouts returns the effective timeouts. tearDown bounds each
+// of the two waits in [hcsTask.close], taskClose the wait in [hcsTask.DeleteExec].
+//
+// DeleteExec waits on the channel close closes, so a taskClose below close's
+// worst case of 2*tearDown would abandon a teardown that is still progressing.
+// When taskClose is unset and tearDown was raised, it is derived to cover that.
+//
+// Empty, unparseable or non-positive values count as unset: this runs at
+// package init, before logging exists, and must not stop the shim starting.
+func resolveTeardownTimeouts(tearDownRaw, taskCloseRaw string) (tearDown, taskClose time.Duration) {
+	tearDown = defaultTearDownTimeout
+	if d, ok := parsePositiveDuration(tearDownRaw); ok {
+		tearDown = d
+	}
+
+	if d, ok := parsePositiveDuration(taskCloseRaw); ok {
+		return tearDown, d
+	}
+	if tearDown <= defaultTearDownTimeout {
+		return tearDown, defaultTaskCloseTimeout
+	}
+	return tearDown, 2*tearDown + defaultTaskCloseTimeout
+}
+
+func parsePositiveDuration(s string) (time.Duration, bool) {
+	if s == "" {
+		return 0, false
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil || d <= 0 {
+		return 0, false
+	}
+	return d, true
+}
+
 func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (shimTask, error) {
 	log.G(ctx).WithField("tid", req.ID).Debug("newHcsStandaloneTask")
 
@@ -536,7 +592,7 @@ func (ht *hcsTask) DeleteExec(ctx context.Context, eid string) (int, uint32, tim
 		// If the shim exits before resources are cleaned up, those resources
 		// will remain locked and untracked, which leads to lingering sandboxes
 		// and container resources like base vhdx.
-		const timeout = 30 * time.Second
+		timeout := taskCloseTimeout
 		entry.WithField(logfields.Timeout, timeout).Trace("waiting for task to be closed")
 		select {
 		case <-time.After(timeout):
@@ -702,8 +758,6 @@ func (ht *hcsTask) close(ctx context.Context) {
 		// method or interface for ht.c operations that we can stub for
 		// testing.
 		if ht.c != nil {
-			const tearDownTimeout = 30 * time.Second
-
 			// Do our best attempt to tear down the container.
 			// TODO: unify timeout select statements and use [ht.c.WaitCtx] and [context.WithTimeout]
 			var werr error
@@ -717,11 +771,17 @@ func (ht *hcsTask) close(ctx context.Context) {
 			if err != nil {
 				entry.WithError(err).Error("failed to shutdown container")
 			} else {
+				shutdownStart := time.Now()
 				t := time.NewTimer(tearDownTimeout)
 				select {
 				case <-ch:
 					err = werr
 					t.Stop()
+					// The number needed to size tearDownTimeout.
+					entry.WithFields(logrus.Fields{
+						logfields.Timeout: tearDownTimeout,
+						"duration":        time.Since(shutdownStart).String(),
+					}).Debug("container shutdown completed")
 					if err != nil {
 						entry.WithError(err).Error("failed to wait for container shutdown")
 					}

cmd/containerd-shim-runhcs-v1/task_hcs_test.go

@@ -550,3 +550,72 @@ func Test_hcsTask_updateWCOWContainerCPUAffinity_XenonNotImplemented(t *testing.
 		t.Fatalf("expected ErrNotImplemented for hypervisor-isolated container, got %v", err)
 	}
 }
+
+func Test_resolveTeardownTimeouts(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		tearDownRaw  string
+		taskCloseRaw string
+		tearDown     time.Duration
+		taskClose    time.Duration
+	}{
+		{
+			name:      "unset keeps the historical defaults",
+			tearDown:  30 * time.Second,
+			taskClose: 30 * time.Second,
+		},
+		{
+			name:        "raising teardown derives a task close that covers it",
+			tearDownRaw: "45m",
+			tearDown:    45 * time.Minute,
+			taskClose:   2*45*time.Minute + 30*time.Second,
+		},
+		{
+			name:         "explicit task close wins over the derived value",
+			tearDownRaw:  "45m",
+			taskCloseRaw: "100m",
+			tearDown:     45 * time.Minute,
+			taskClose:    100 * time.Minute,
+		},
+		{
+			name:         "task close alone is honoured",
+			taskCloseRaw: "5m",
+			tearDown:     30 * time.Second,
+			taskClose:    5 * time.Minute,
+		},
+		{
+			name:        "lowering teardown does not derive",
+			tearDownRaw: "10s",
+			tearDown:    10 * time.Second,
+			taskClose:   30 * time.Second,
+		},
+		{
+			name:         "malformed and negative values fall back",
+			tearDownRaw:  "soon",
+			taskCloseRaw: "-5m",
+			tearDown:     30 * time.Second,
+			taskClose:    30 * time.Second,
+		},
+		{
+			name:        "zero is not a positive duration",
+			tearDownRaw: "0s",
+			tearDown:    30 * time.Second,
+			taskClose:   30 * time.Second,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			tearDown, taskClose := resolveTeardownTimeouts(tc.tearDownRaw, tc.taskCloseRaw)
+			if tearDown != tc.tearDown {
+				t.Errorf("tearDown: expected %v, got %v", tc.tearDown, tearDown)
+			}
+			if taskClose != tc.taskClose {
+				t.Errorf("taskClose: expected %v, got %v", tc.taskClose, taskClose)
+			}
+			// The invariant the derivation exists to protect: DeleteExec must
+			// not give up while close() may still be making progress.
+			if tc.taskCloseRaw == "" && taskClose <= 2*tearDown && tearDown > defaultTearDownTimeout {
+				t.Errorf("derived taskClose %v does not cover close()'s worst case of 2*%v", taskClose, tearDown)
+			}
+		})
+	}
+}