Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions pkg/artifact/blob/s3/s3test/fault.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ const (
// DropBeforeCommit closes the connection without applying the request.
DropBeforeCommit

// Gate holds a fully received request until released, even if its client disconnects.
Gate

// Delay sleeps before handling the request normally.
Delay
)
Expand Down Expand Up @@ -51,6 +54,13 @@ type Fault struct {
Status int32
Code string

// Entered and Release synchronize a [Gate]; Done closes after the handler finishes.
// A Gate applies to exactly one request: [Server.Inject] ignores Times for it, so the channels
// are closed at most once.
Entered chan<- struct{}
Release <-chan struct{}
Done chan<- struct{}

// Delay is the pause for [Delay].
Delay time.Duration
}
Expand Down Expand Up @@ -127,6 +137,9 @@ func (f *Fault) status() (int, string) {
func (s *Server) Inject(fault Fault) {
defer perf.Track(nil, "s3test.Server.Inject")()

if fault.Action == Gate {
fault.Times = 1
}
s.faults.add(&fault)
}

Expand Down
14 changes: 13 additions & 1 deletion pkg/artifact/blob/s3/s3test/server.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package s3test

import (
"bytes"
"io"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -113,6 +114,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
if fault != nil {
s.rec.update(index, func(rr *Request) { rr.Faulted = true })
}
if fault != nil && fault.Action == Gate {
defer close(fault.Done)
body, err := io.ReadAll(r.Body)
if err != nil {
writeError(w, http.StatusBadRequest, "IncompleteBody", err.Error())
return
}
r.Body = io.NopCloser(bytes.NewReader(body))
close(fault.Entered)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
<-fault.Release
}
rw := &statusWriter{ResponseWriter: w}
if fault != nil && s.applyEarlyFault(rw, r, fault) {
s.finish(index, rw)
Expand Down Expand Up @@ -149,7 +161,7 @@ func (s *Server) applyEarlyFault(w *statusWriter, r *http.Request, fault *Fault)
return true
case Delay:
sleep(r.Context(), fault.Delay)
case DropAfterCommit:
case DropAfterCommit, Gate:
}
return false
}
Expand Down
32 changes: 32 additions & 0 deletions pkg/artifact/blob/s3/s3test/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http"
"strconv"
"strings"
"sync"
"testing"
"time"

Expand Down Expand Up @@ -439,3 +440,34 @@ func TestAttributesWrapTagFailuresWithSharedSentinels(t *testing.T) {
})
}
}

// A Gate applies to one request even when Times asks for more, so its channels close once.
func TestGateAppliesToExactlyOneRequest(t *testing.T) {
srv := NewServer(t)
entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
// Registered after NewServer, so it runs first and the server's Close cannot wait on a held request.
t.Cleanup(unblock)
srv.Inject(Fault{Op: OpPutObject, Action: Gate, Times: -1, Entered: entered, Release: release, Done: done})
client := noRetryClient(srv)
first := make(chan error, 1)
go func() {
_, err := putObject(t, client, srv, "gated", "x")
first <- err
}()
select {
case <-entered:
case err := <-first:
require.Fail(t, "the first request finished before reaching the gate", "error: %v", err)
}
_, err := putObject(t, client, srv, "ungated", "y")
require.NoError(t, err, "a second matching request is not gated")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
unblock()
require.NoError(t, <-first)
<-done
for _, key := range []string{"gated", "ungated"} {
_, ok := srv.Object(key)
assert.True(t, ok, key)
}
}
10 changes: 6 additions & 4 deletions pkg/artifact/s3/delegated_mutation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) {
for _, operation := range []string{"sign", "evidence", "delete"} {
t.Run(operation, func(t *testing.T) {
ctx := context.Background()
spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(time.Second))
spec := signedSpec(t, noneSpec()).withOpts(WithLockWait(100 * time.Millisecond))
repo := newRepo(t, spec)
pinned := publish(t, repo, &prepareSpec{}, artifact.PublishOptions{}).Pinned
subject := recordSubject(t, repo, &pinned)
Expand Down Expand Up @@ -47,9 +47,11 @@ func TestDelegatedMutationsAcquireCanonicalClaim(t *testing.T) {
held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "direct"})
require.NoError(t, err)
before := objectSnapshot(t, spec.fake)
wait, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
defer cancel()
require.ErrorIs(t, mutate(wait, delegated), context.DeadlineExceeded)
// Use the repository wait limit, not cancellation: an interrupted PUT can still
// arrive after the holder is released and leave an ambiguous claim.
err = mutate(ctx, delegated)
require.ErrorIs(t, err, errUtils.ErrArtifactConflict)
require.NotErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous)
assert.Equal(t, before, objectSnapshot(t, spec.fake), "failed acquisition must not mutate target data or evidence")
require.NoError(t, held.Release(ctx))
require.NoError(t, mutate(ctx, delegated))
Expand Down
51 changes: 51 additions & 0 deletions pkg/artifact/s3/lock_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -459,3 +459,54 @@ func TestLock_RefusesAClaimLargerThanItCanBeRead(t *testing.T) {
require.ErrorIs(t, err, errUtils.ErrArtifactTooLarge)
assert.Empty(t, lockObjects(spec.fake), "no claim object was created")
}

// A canceled client cannot establish that a request already received by S3 has stopped.
func TestLock_CanceledContenderMayCreateClaimAfterForeignRelease(t *testing.T) {
spec := noneSpec().fresh(t)
repo := newRepo(t, spec)
ctx := context.Background()
held, err := repo.Lock(ctx, artifact.LockRequest{Resource: lockResource, Owner: "holder"})
require.NoError(t, err)
entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
t.Cleanup(unblock)
spec.fake.Inject(s3test.Fault{Op: s3test.OpPutObject, KeyPrefix: layout.LockPath(lockResource), Action: s3test.Gate, Entered: entered, Release: release, Done: done})
attemptCtx, cancel := context.WithCancel(ctx)
t.Cleanup(cancel)
result := make(chan error, 1)
go func() {
_, err := repo.Lock(attemptCtx, artifact.LockRequest{Resource: lockResource, Owner: "contender"})
result <- err
}()
select {
case <-entered:
case err := <-result:
t.Fatalf("lock completed before entering gate: %v", err)
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for claim to enter gate")
}
cancel()
select {
case err = <-result:
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for canceled claim")
}
require.ErrorIs(t, err, context.Canceled)
require.ErrorIs(t, err, errUtils.ErrArtifactPublicationAmbiguous)
require.NoError(t, held.Release(ctx))
unblock()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for gated request to finish")
}
keys := lockObjects(spec.fake)
require.Len(t, keys, 1, "the late request can create its claim after release")
object, ok := spec.fake.Object(keys[0])
require.True(t, ok)
var doc claimDoc
require.NoError(t, json.Unmarshal(object.Data, &doc))
assert.Equal(t, "contender", doc.Owner)
assert.Contains(t, err.Error(), doc.Token, "the ambiguity diagnostic identifies the late claim")
}
Loading