From 1c7885b141f80c6722b9b14286a8186ea7e36d37 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 17 Nov 2026 16:06:43 -0600 Subject: [PATCH 1/4] test(artifacts): avoid ambiguous claim cancellation in delegation test --- pkg/artifact/blob/s3/s3test/fault.go | 8 +++++ pkg/artifact/blob/s3/s3test/server.go | 14 ++++++++- pkg/artifact/s3/delegated_mutation_test.go | 10 +++--- pkg/artifact/s3/lock_test.go | 36 ++++++++++++++++++++++ 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/pkg/artifact/blob/s3/s3test/fault.go b/pkg/artifact/blob/s3/s3test/fault.go index 5a6e00bd892..7aa7b0e12f3 100644 --- a/pkg/artifact/blob/s3/s3test/fault.go +++ b/pkg/artifact/blob/s3/s3test/fault.go @@ -24,6 +24,9 @@ const ( // DropBeforeCommit closes the connection without applying the request. DropBeforeCommit + // Gate holds a fully received request until released, even if its client disconnects. + Gate + // Delay sleeps before handling the request normally. Delay ) @@ -51,6 +54,11 @@ type Fault struct { Status int32 Code string + // Entered and Release synchronize a [Gate]; Done closes after the handler finishes. + Entered chan<- struct{} + Release <-chan struct{} + Done chan<- struct{} + // Delay is the pause for [Delay]. Delay time.Duration } diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go index 945e0937fed..e494873c72b 100644 --- a/pkg/artifact/blob/s3/s3test/server.go +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -1,6 +1,7 @@ package s3test import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -113,6 +114,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { if fault != nil { s.rec.update(index, func(rr *Request) { rr.Faulted = true }) } + if fault != nil && fault.Action == Gate { + body, err := io.ReadAll(r.Body) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + close(fault.Entered) + <-fault.Release + defer close(fault.Done) + } rw := &statusWriter{ResponseWriter: w} if fault != nil && s.applyEarlyFault(rw, r, fault) { s.finish(index, rw) @@ -149,7 +161,7 @@ func (s *Server) applyEarlyFault(w *statusWriter, r *http.Request, fault *Fault) return true case Delay: sleep(r.Context(), fault.Delay) - case DropAfterCommit: + case DropAfterCommit, Gate: } return false } diff --git a/pkg/artifact/s3/delegated_mutation_test.go b/pkg/artifact/s3/delegated_mutation_test.go index 2a25afe7d9b..7db8d9a2cfa 100644 --- a/pkg/artifact/s3/delegated_mutation_test.go +++ b/pkg/artifact/s3/delegated_mutation_test.go @@ -17,7 +17,7 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) { for _, operation := range []string{"sign", "evidence", "delete"} { t.Run(operation, func(t *testing.T) { ctx := context.Background() - spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(time.Second)) + spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(100 * time.Millisecond)) repo := newRepo(t, spec) pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned subject := recordSubject(t, repo, &pinned) @@ -47,9 +47,11 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) { held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "direct"}) require.NoError(t, err) before := objectSnapshot(t, spec.fake) - wait, cancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer cancel() - require.ErrorIs(t, mutate(wait, delegated), context.DeadlineExceeded) + // Use the repository wait limit, not cancellation: an interrupted PUT can still + // arrive after the holder is released and leave an ambiguous claim. + err = mutate(ctx, delegated) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NotErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) assert.Equal(t, before, objectSnapshot(t, spec.fake), "failed acquisition must not mutate target data or evidence") require.NoError(t, held.Release(ctx)) require.NoError(t, mutate(ctx, delegated)) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 5633a050784..9785d300744 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -459,3 +459,39 @@ func TestLock_RefusesAClaimLargerThanItCanBeRead(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) assert.Empty(t, lockObjects(spec.fake), "no claim object was created") } + +// A canceled client cannot establish that a request already received by S3 has stopped. +func TestLock_CanceledContenderMayCreateClaimAfterForeignRelease(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + ctx := context.Background() + held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "holder"}) + require.NoError(t, err) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.Gate, Entered: entered, Release: release, Done: done}) + attemptCtx, cancel := context.WithCancel(ctx) + result := make(chan error, 1) + go func() { + _, err := repo.Lock(attemptCtx, artifact.LockRequest{Resource: lockResource, Owner: "contender"}) + result <- err + }() + <-entered + cancel() + err = <-result + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + require.NoError(t, held.Release(ctx)) + unblock() + <-done + keys := lockObjects(spec.fake) + require.Len(t, keys, 1, "the late request can create its claim after release") + object, ok := spec.fake.Object(keys[0]) + require.True(t, ok) + var doc claimDoc + require.NoError(t, json.Unmarshal(object.Data, &doc)) + assert.Equal(t, "contender", doc.Owner) + assert.Contains(t, err.Error(), doc.Token, "the ambiguity diagnostic identifies the late claim") +} From e1c8ba9c1d3505b2595781a8cc6c950725c73fa0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 17 Nov 2026 17:44:31 -0600 Subject: [PATCH 2/4] test(artifacts): bound late claim synchronization --- pkg/artifact/s3/lock_test.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 9785d300744..b6e2cf969f0 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -473,19 +473,34 @@ func TestLock_CanceledContenderMayCreateClaimAfterForeignRelease(t *testing.T) { t.Cleanup(unblock) spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.Gate, Entered: entered, Release: release, Done: done}) attemptCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) result := make(chan error, 1) go func() { _, err := repo.Lock(attemptCtx, artifact.LockRequest{Resource: lockResource, Owner: "contender"}) result <- err }() - <-entered + select { + case <-entered: + case err := <-result: + t.Fatalf("lock completed before entering gate: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for claim to enter gate") + } cancel() - err = <-result + select { + case err = <-result: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for canceled claim") + } require.ErrorIs(t, err, context.Canceled) require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) require.NoError(t, held.Release(ctx)) unblock() - <-done + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for gated request to finish") + } keys := lockObjects(spec.fake) require.Len(t, keys, 1, "the late request can create its claim after release") object, ok := spec.fake.Object(keys[0]) From c71ccf12c37cbec7c33514c152be5ef01ef2bbf7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 17 Nov 2026 19:33:43 -0600 Subject: [PATCH 3/4] test(artifacts): make s3test Gate single-request and always signal Done Close Done before reading the body so a truncated gated PUT still signals completion, and force a Gate fault to apply to exactly one request so its channels cannot be closed twice. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/s3/s3test/fault.go | 5 +++++ pkg/artifact/blob/s3/s3test/server.go | 2 +- pkg/artifact/blob/s3/s3test/server_test.go | 23 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pkg/artifact/blob/s3/s3test/fault.go b/pkg/artifact/blob/s3/s3test/fault.go index 7aa7b0e12f3..ba20f220432 100644 --- a/pkg/artifact/blob/s3/s3test/fault.go +++ b/pkg/artifact/blob/s3/s3test/fault.go @@ -55,6 +55,8 @@ type Fault struct { Code string // Entered and Release synchronize a [Gate]; Done closes after the handler finishes. + // A Gate applies to exactly one request: [Server.Inject] ignores Times for it, so the channels + // are closed at most once. Entered chan<- struct{} Release <-chan struct{} Done chan<- struct{} @@ -135,6 +137,9 @@ func (f *Fault) status() (int, string) { func (s *Server) Inject(fault Fault) { defer perf.Track(nil, "s3test.Server.Inject")() + if fault.Action == Gate { + fault.Times = 1 + } s.faults.add(&fault) } diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go index e494873c72b..331cc84d817 100644 --- a/pkg/artifact/blob/s3/s3test/server.go +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -115,6 +115,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { s.rec.update(index, func(rr *Request) { rr.Faulted = true }) } if fault != nil && fault.Action == Gate { + defer close(fault.Done) body, err := io.ReadAll(r.Body) if err != nil { writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) @@ -123,7 +124,6 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { r.Body = io.NopCloser(bytes.NewReader(body)) close(fault.Entered) <-fault.Release - defer close(fault.Done) } rw := &statusWriter{ResponseWriter: w} if fault != nil && s.applyEarlyFault(rw, r, fault) { diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 0360c578f70..d499cf9bf5b 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -439,3 +439,26 @@ func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) { }) } } + +// A Gate applies to one request even when Times asks for more, so its channels close once. +func TestGateAppliesToExactlyOneRequest(t *testing.T) { + srv := NewServer(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + srv.Inject(Fault{Op: OpPutObject, Action: Gate, Times: -1, Entered: entered, Release: release, Done: done}) + client := noRetryClient(srv) + first := make(chan error, 1) + go func() { + _, err := putObject(t, client, srv, "gated", "x") + first <- err + }() + <-entered + _, err := putObject(t, client, srv, "ungated", "y") + require.NoError(t, err, "a second matching request is not gated") + close(release) + require.NoError(t, <-first) + <-done + for _, key := range []string{"gated", "ungated"} { + _, ok := srv.Object(key) + assert.True(t, ok, key) + } +} From ad5fdc2a6b0a1780cf743c7e45d890fe59e27ba3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Tue, 17 Nov 2026 19:42:07 -0600 Subject: [PATCH 4/4] test(artifacts): release gated request on test failure and fail fast Register an idempotent release cleanup so a failing assertion cannot hang the server Close, and observe the first PUT result while waiting for the gate so an early failure ends the test directly. Co-Authored-By: Claude Sonnet 5.5 --- pkg/artifact/blob/s3/s3test/server_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index d499cf9bf5b..e6b77b61160 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "sync" "testing" "time" @@ -444,6 +445,10 @@ func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) { func TestGateAppliesToExactlyOneRequest(t *testing.T) { srv := NewServer(t) entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + // Registered after NewServer, so it runs first and the server's Close cannot wait on a held request. + t.Cleanup(unblock) srv.Inject(Fault{Op: OpPutObject, Action: Gate, Times: -1, Entered: entered, Release: release, Done: done}) client := noRetryClient(srv) first := make(chan error, 1) @@ -451,10 +456,14 @@ func TestGateAppliesToExactlyOneRequest(t *testing.T) { _, err := putObject(t, client, srv, "gated", "x") first <- err }() - <-entered + select { + case <-entered: + case err := <-first: + require.Fail(t, "the first request finished before reaching the gate", "error: %v", err) + } _, err := putObject(t, client, srv, "ungated", "y") require.NoError(t, err, "a second matching request is not gated") - close(release) + unblock() require.NoError(t, <-first) <-done for _, key := range []string{"gated", "ungated"} {