diff --git a/pkg/artifact/blob/s3/s3test/fault.go b/pkg/artifact/blob/s3/s3test/fault.go index 5a6e00bd89..ba20f22043 100644 --- a/pkg/artifact/blob/s3/s3test/fault.go +++ b/pkg/artifact/blob/s3/s3test/fault.go @@ -24,6 +24,9 @@ const ( // DropBeforeCommit closes the connection without applying the request. DropBeforeCommit + // Gate holds a fully received request until released, even if its client disconnects. + Gate + // Delay sleeps before handling the request normally. Delay ) @@ -51,6 +54,13 @@ type Fault struct { Status int32 Code string + // Entered and Release synchronize a [Gate]; Done closes after the handler finishes. + // A Gate applies to exactly one request: [Server.Inject] ignores Times for it, so the channels + // are closed at most once. + Entered chan<- struct{} + Release <-chan struct{} + Done chan<- struct{} + // Delay is the pause for [Delay]. Delay time.Duration } @@ -127,6 +137,9 @@ func (f *Fault) status() (int, string) { func (s *Server) Inject(fault Fault) { defer perf.Track(nil, "s3test.Server.Inject")() + if fault.Action == Gate { + fault.Times = 1 + } s.faults.add(&fault) } diff --git a/pkg/artifact/blob/s3/s3test/server.go b/pkg/artifact/blob/s3/s3test/server.go index 945e0937fe..331cc84d81 100644 --- a/pkg/artifact/blob/s3/s3test/server.go +++ b/pkg/artifact/blob/s3/s3test/server.go @@ -1,6 +1,7 @@ package s3test import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -113,6 +114,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { if fault != nil { s.rec.update(index, func(rr *Request) { rr.Faulted = true }) } + if fault != nil && fault.Action == Gate { + defer close(fault.Done) + body, err := io.ReadAll(r.Body) + if err != nil { + writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error()) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + close(fault.Entered) + <-fault.Release + } rw := &statusWriter{ResponseWriter: w} if fault != nil && s.applyEarlyFault(rw, r, fault) { s.finish(index, rw) @@ -149,7 +161,7 @@ func (s *Server) applyEarlyFault(w *statusWriter, r *http.Request, fault *Fault) return true case Delay: sleep(r.Context(), fault.Delay) - case DropAfterCommit: + case DropAfterCommit, Gate: } return false } diff --git a/pkg/artifact/blob/s3/s3test/server_test.go b/pkg/artifact/blob/s3/s3test/server_test.go index 0360c578f7..e6b77b6116 100644 --- a/pkg/artifact/blob/s3/s3test/server_test.go +++ b/pkg/artifact/blob/s3/s3test/server_test.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "sync" "testing" "time" @@ -439,3 +440,34 @@ func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) { }) } } + +// A Gate applies to one request even when Times asks for more, so its channels close once. +func TestGateAppliesToExactlyOneRequest(t *testing.T) { + srv := NewServer(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + // Registered after NewServer, so it runs first and the server's Close cannot wait on a held request. + t.Cleanup(unblock) + srv.Inject(Fault{Op: OpPutObject, Action: Gate, Times: -1, Entered: entered, Release: release, Done: done}) + client := noRetryClient(srv) + first := make(chan error, 1) + go func() { + _, err := putObject(t, client, srv, "gated", "x") + first <- err + }() + select { + case <-entered: + case err := <-first: + require.Fail(t, "the first request finished before reaching the gate", "error: %v", err) + } + _, err := putObject(t, client, srv, "ungated", "y") + require.NoError(t, err, "a second matching request is not gated") + unblock() + require.NoError(t, <-first) + <-done + for _, key := range []string{"gated", "ungated"} { + _, ok := srv.Object(key) + assert.True(t, ok, key) + } +} diff --git a/pkg/artifact/s3/delegated_mutation_test.go b/pkg/artifact/s3/delegated_mutation_test.go index 2a25afe7d9..7db8d9a2cf 100644 --- a/pkg/artifact/s3/delegated_mutation_test.go +++ b/pkg/artifact/s3/delegated_mutation_test.go @@ -17,7 +17,7 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) { for _, operation := range []string{"sign", "evidence", "delete"} { t.Run(operation, func(t *testing.T) { ctx := context.Background() - spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(time.Second)) + spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(100 * time.Millisecond)) repo := newRepo(t, spec) pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned subject := recordSubject(t, repo, &pinned) @@ -47,9 +47,11 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) { held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "direct"}) require.NoError(t, err) before := objectSnapshot(t, spec.fake) - wait, cancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer cancel() - require.ErrorIs(t, mutate(wait, delegated), context.DeadlineExceeded) + // Use the repository wait limit, not cancellation: an interrupted PUT can still + // arrive after the holder is released and leave an ambiguous claim. + err = mutate(ctx, delegated) + require.ErrorIs(t, err, errUtils.ErrArtifactConflict) + require.NotErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) assert.Equal(t, before, objectSnapshot(t, spec.fake), "failed acquisition must not mutate target data or evidence") require.NoError(t, held.Release(ctx)) require.NoError(t, mutate(ctx, delegated)) diff --git a/pkg/artifact/s3/lock_test.go b/pkg/artifact/s3/lock_test.go index 5633a05078..b6e2cf969f 100644 --- a/pkg/artifact/s3/lock_test.go +++ b/pkg/artifact/s3/lock_test.go @@ -459,3 +459,54 @@ func TestLock_RefusesAClaimLargerThanItCanBeRead(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge) assert.Empty(t, lockObjects(spec.fake), "no claim object was created") } + +// A canceled client cannot establish that a request already received by S3 has stopped. +func TestLock_CanceledContenderMayCreateClaimAfterForeignRelease(t *testing.T) { + spec := noneSpec().fresh(t) + repo := newRepo(t, spec) + ctx := context.Background() + held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "holder"}) + require.NoError(t, err) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.Gate, Entered: entered, Release: release, Done: done}) + attemptCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + result := make(chan error, 1) + go func() { + _, err := repo.Lock(attemptCtx, artifact.LockRequest{Resource: lockResource, Owner: "contender"}) + result <- err + }() + select { + case <-entered: + case err := <-result: + t.Fatalf("lock completed before entering gate: %v", err) + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for claim to enter gate") + } + cancel() + select { + case err = <-result: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for canceled claim") + } + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous) + require.NoError(t, held.Release(ctx)) + unblock() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for gated request to finish") + } + keys := lockObjects(spec.fake) + require.Len(t, keys, 1, "the late request can create its claim after release") + object, ok := spec.fake.Object(keys[0]) + require.True(t, ok) + var doc claimDoc + require.NoError(t, json.Unmarshal(object.Data, &doc)) + assert.Equal(t, "contender", doc.Owner) + assert.Contains(t, err.Error(), doc.Token, "the ambiguity diagnostic identifies the late claim") +}