diff --git a/transport/http2/client.go b/transport/http2/client.go index 0bda5aec..9690edcd 100644 --- a/transport/http2/client.go +++ b/transport/http2/client.go @@ -5,63 +5,26 @@ import ( "crypto/tls" "fmt" "io" - "math" - "math/rand/v2" "net" "net/http" "runtime" "sync" - "sync/atomic" "time" utls "github.com/refraction-networking/utls" sharedconfig "github.com/nange/easyss/v3/config" - "github.com/nange/easyss/v3/log" "github.com/nange/easyss/v3/stats" "github.com/nange/easyss/v3/transport" ) -type transportSlot struct { - t *http.Transport - active atomic.Int32 - heavy atomic.Int32 // number of active heavy streams (>= HeavyStreamThreshold bytes) - // bytesRecv is the cumulative downloaded bytes across streams on this - // slot; sampled by the health loop to estimate recent throughput. - bytesRecv atomic.Int64 - // degraded marks a slot whose download throughput stays below the - // degraded threshold while hosting heavy streams; new streams avoid it - // and its idle connection is retired early. - degraded atomic.Bool - - // Connection rotation state. - expireAt atomic.Int64 // unix nano deadline of the current connection (dial time + lifetime + jitter) - connBytesRecv atomic.Int64 // bytes downloaded over the current connection - // expiring marks a slot whose connection exceeded the lifetime or bytes - // limit; new streams avoid it and its idle connection is closed so the - // next stream dials a fresh one. Cleared when a new connection is - // established or the rotation completes. - expiring atomic.Bool - - // Health-loop state, touched only from the health loop goroutine. - lastBytes int64 - lastHeavy int // last observed heavy count, tracks heavy 0->1 transitions - lowCycles int - recoverCycles int -} - +// HTTP2Transport is a facade over the HTTP/2 client machinery: streams are +// mapped onto connections by slotScheduler, and the per-connection state +// (degradation, rotation) is driven by slotLifecycle. This type only wires +// the two together and speaks HTTP. type HTTP2Transport struct { - slots []*transportSlot // pre-allocated and initialized to maxSlots - liveCount atomic.Int32 // number of currently active slots (0..maxSlots) - maxSlots int - threshold int32 - prioritySlots int // number of priority slots (0..prioritySlots-1) - bulkThreshold int32 - mu sync.RWMutex // protects slot retire (shrink) and grow; RLock protects stream assignment - - // Connection rotation limits. - connLifetime time.Duration // max age of a connection before rotation - connMaxBytes int64 // max bytes per connection before rotation + sched *slotScheduler + lifecycle *slotLifecycle serverURL string @@ -103,8 +66,6 @@ func New(cfg Config) (*HTTP2Transport, error) { prioritySlots = maxSlots } - bulkThreshold := threshold * 2 - timeout := cfg.Timeout if timeout <= 0 { timeout = 30 * time.Second @@ -133,19 +94,20 @@ func New(cfg Config) (*HTTP2Transport, error) { slots[i] = newSlot(cfg.TLSConfig, timeout, dialCtx, connLifetime) } + sched := newScheduler(maxSlots, slots, threshold, prioritySlots) + tr := &HTTP2Transport{ - slots: slots, - maxSlots: maxSlots, - threshold: threshold, - prioritySlots: prioritySlots, - bulkThreshold: bulkThreshold, - connLifetime: connLifetime, - connMaxBytes: connMaxBytes, - serverURL: cfg.ServerURL, - ctx: ctx, - cancel: cancel, + sched: sched, + lifecycle: &slotLifecycle{ + sched: sched, + connLifetime: connLifetime, + connMaxBytes: connMaxBytes, + }, + serverURL: cfg.ServerURL, + ctx: ctx, + cancel: cancel, } - go tr.healthLoop() + go tr.lifecycle.run(ctx) return tr, nil } @@ -206,9 +168,7 @@ func newSlot(utlsCfg *utls.Config, timeout time.Duration, dialContext func(conte // A new connection resets the rotation state: the lifetime // deadline (with per-connection jitter), bytes carried and the // expiring mark all start fresh. - slot.expireAt.Store(time.Now().Add(rotationLifetime(connLifetime)).UnixNano()) - slot.connBytesRecv.Store(0) - slot.expiring.Store(false) + slot.resetConn(connLifetime) return uconn, nil }, } @@ -230,17 +190,17 @@ func (t *HTTP2Transport) Open(ctx context.Context, req transport.OpenRequest) (t stats.RecordStreamOpened() - t.maybeGrowSlots(req.HighPriority) + t.sched.grow(req.HighPriority) - t.mu.RLock() - slot := t.selectSlot(req.HighPriority) + t.sched.mu.RLock() + slot := t.sched.pick(req.HighPriority) slot.active.Add(1) if req.HighPriority { stats.RecordStreamOpenedPriority() } else { stats.RecordStreamOpenedBulk() } - t.mu.RUnlock() + t.sched.mu.RUnlock() parentCtx := ctx ctx, cancel := context.WithCancel(parentCtx) @@ -317,380 +277,24 @@ func (t *HTTP2Transport) Open(ctx context.Context, req transport.OpenRequest) (t return stream, nil } -func (t *HTTP2Transport) selectSlot(highPriority bool) *transportSlot { - if highPriority && t.prioritySlots > 0 { - slot := t.leastActiveSlotRange(0, t.prioritySlots) - if slot == nil || slot.active.Load() >= t.threshold { - stats.RecordPriorityFallback() - slot = t.leastActiveSlotRange(t.prioritySlots, int(t.liveCount.Load())) - } - return slot - } - - slot := t.leastActiveSlotRange(t.prioritySlots, int(t.liveCount.Load())) - if slot == nil || slot.active.Load() >= t.bulkThreshold { - stats.RecordBulkFallback() - slot = t.leastActiveSlotRange(0, t.prioritySlots) - } - return slot -} - -func (t *HTTP2Transport) leastActiveSlotRange(start, end int) *transportSlot { - live := int(t.liveCount.Load()) - if live == 0 { - return t.slots[0] - } - if end > live { - end = live - } - if start >= end { - start = 0 - end = live - } - // Prefer healthy slots, then slots without heavy streams, and only fall - // back to degraded or expiring ones when nothing else is available: - // - a heavy stream monopolizes the connection's TCP window, so a new - // stream sharing that slot is dragged down together with it (TCP - // head-of-line blocking under packet loss); - // - a degraded slot already proved persistently low throughput, so it - // is avoided unless it is the only option left; - // - an expiring slot is due for connection rotation, so new streams - // avoid it unless nothing else is available. - passes := []struct { - skipHeavy bool - skipDegraded bool - skipExpiring bool - }{ - {true, true, true}, // healthy slots - {false, true, true}, // heavy but not degraded/expiring - {false, false, true}, // degraded but not expiring - {false, false, false}, - } - var best *transportSlot - for _, p := range passes { - var min int32 = math.MaxInt32 - for i := start; i < end; i++ { - s := t.slots[i] - if p.skipHeavy && s.heavy.Load() > 0 { - continue - } - if p.skipDegraded && s.degraded.Load() { - continue - } - if p.skipExpiring && s.expiring.Load() { - continue - } - if a := s.active.Load(); a < min { - best, min = s, a - } - } - if best != nil { - return best - } - } - return t.slots[0] -} - -// maybeGrowSlots checks whether the live slots that a new stream would -// actually use are all at or above the threshold, and if so, activates one -// more slot (up to maxSlots). Uses double-checked locking. -func (t *HTTP2Transport) maybeGrowSlots(highPriority bool) { - live := t.liveCount.Load() - if int(live) >= t.maxSlots { - return - } - - thresh := t.threshold - start, end := int32(0), live - if highPriority && t.prioritySlots > 0 { - end = int32(t.prioritySlots) - if end > live { - end = live - } - } else if t.prioritySlots > 0 { - start = int32(t.prioritySlots) - thresh = t.bulkThreshold - } - - if live > 0 { - if start >= end { - return - } - if !t.shouldGrow(start, end, thresh) { - return - } - } - - // All eligible slots in range are at or above threshold — try to grow - // under lock. - t.mu.Lock() - defer t.mu.Unlock() - - // Double-check after acquiring the lock. - live = t.liveCount.Load() - if int(live) >= t.maxSlots { - return - } - start2, end2 := int32(0), live - if highPriority && t.prioritySlots > 0 { - end2 = int32(t.prioritySlots) - if end2 > live { - end2 = live - } - } else if t.prioritySlots > 0 { - start2 = int32(t.prioritySlots) - } - if live > 0 { - if start2 >= end2 { - return - } - if !t.shouldGrow(start2, end2, thresh) { - return - } - } - - // On first activation, start with 2 connections for better initial throughput, - // since typical web browsing generates >8 concurrent streams. - // Falls back to 1 when maxSlots is 1. - if live == 0 && t.maxSlots >= 2 { - t.liveCount.Add(2) - } else { - t.liveCount.Add(1) - } -} - -// shouldGrow reports whether the range [start,end) needs one more live -// slot: every slot a new stream would actually use is at or above the -// threshold. Heavy, degraded and expiring slots are skipped — a new stream -// avoids them, so a heavy slot hosting a single download must not block -// growing more connections. A range with no eligible slot at all also -// grows, since new streams then fall back onto heavy/degraded slots and -// deserve a fresh connection. -func (t *HTTP2Transport) shouldGrow(start, end, thresh int32) bool { - for i := start; i < end; i++ { - s := t.slots[i] - if s.heavy.Load() > 0 || s.degraded.Load() || s.expiring.Load() { - continue - } - if s.active.Load() < thresh { - return false - } - } - return true -} - func (t *HTTP2Transport) CloseIdle() { // Close idle TCP connections on all slots (no lock needed). - for _, s := range t.slots { + for _, s := range t.sched.slots { s.t.CloseIdleConnections() } // Shrink liveCount by retiring idle slots (any position, swap-remove). - t.mu.Lock() - defer t.mu.Unlock() - for t.shrinkIdleLocked() { - } -} - -// shrinkIdleLocked retires one idle slot (active==0) from liveCount, -// swap-removing it to the end. Caller must hold t.mu. Returns false when -// no idle slot remains. -func (t *HTTP2Transport) shrinkIdleLocked() bool { - live := int(t.liveCount.Load()) - for i := 0; i < live; i++ { - if t.slots[i].active.Load() != 0 { - continue - } - last := live - 1 - if i != last { - t.slots[i], t.slots[last] = t.slots[last], t.slots[i] - } - t.liveCount.Add(-1) - return true - } - return false -} - -// healthLoop periodically samples slot health: download throughput feeds -// the degraded detector, connection age/bytes feed rotation, and idle -// degraded slots are retired. It runs until the transport is closed -// (t.ctx cancelled). -func (t *HTTP2Transport) healthLoop() { - interval := sharedconfig.HealthCheckInterval - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - t.evaluateHealth(interval) - case <-t.ctx.Done(): - return - } - } -} - -func (t *HTTP2Transport) evaluateHealth(interval time.Duration) { - // A congested link (high RTT) makes every connection slow: marking or - // retiring slots then only adds handshake churn without recovering - // anything, so degraded detection is gated on a healthy RTT. Rotation, - // on the other hand, is exactly what a throttled connection needs and - // runs regardless of RTT. - linkOK := stats.Collect().AvgRTT() <= sharedconfig.DegradedMaxRTT - - live := int(t.liveCount.Load()) - for i := 0; i < live; i++ { - s := t.slots[i] - t.evaluateSlotHealth(i, s, interval, linkOK) - t.evaluateRotation(i, s) - if linkOK && s.degraded.Load() && s.active.Load() == 0 { - t.retireSlot(s) - // retireSlot swap-removes the slot to the end and shrinks - // liveCount; re-evaluate the slot swapped into position i. - live-- - i-- - } - } -} - -// evaluateSlotHealth updates one slot's degraded state from its recent -// download throughput. Only slots hosting heavy streams are considered — -// idle or short-lived slots naturally carry zero throughput. The mark is -// set after DegradedPersistCycles consecutive slow intervals and cleared -// after DegradedRecoverCycles healthy ones. -func (t *HTTP2Transport) evaluateSlotHealth(idx int, s *transportSlot, interval time.Duration, linkOK bool) { - if s.heavy.Load() == 0 { - s.lastHeavy = 0 - return - } - if s.lastHeavy == 0 { - // First heavy stream on this slot since it went idle: bytes - // transferred by earlier small streams must not skew the first - // sample, so reset the baseline and skip this interval. - s.lastHeavy = 1 - s.lastBytes = s.bytesRecv.Load() - s.lowCycles = 0 - s.recoverCycles = 0 - return - } - if !linkOK { - // Congested link: low throughput is a link property, not evidence - // that this particular connection is broken. Advance the baseline - // so the first healthy interval starts from a clean sample, and - // freeze the counters. - s.lastBytes = s.bytesRecv.Load() - return - } - - now := s.bytesRecv.Load() - perSec := int64(interval / time.Second) - if perSec <= 0 { - perSec = 1 - } - throughput := (now - s.lastBytes) / perSec - s.lastBytes = now - - if throughput >= int64(sharedconfig.DegradedThroughputThreshold) { - s.lowCycles = 0 - if s.degraded.Load() { - s.recoverCycles++ - if s.recoverCycles >= sharedconfig.DegradedRecoverCycles { - s.degraded.Store(false) - s.recoverCycles = 0 - log.Info("[TRANSPORT] slot recovered", "slot", idx, "throughput_kb_s", throughput/1024) - } - } - return - } - - s.recoverCycles = 0 - s.lowCycles++ - if s.lowCycles >= sharedconfig.DegradedPersistCycles && !s.degraded.Load() { - s.degraded.Store(true) - s.lowCycles = 0 - stats.RecordSlotDegraded() - log.Info("[TRANSPORT] slot degraded", "slot", idx, "throughput_kb_s", throughput/1024) - } -} - -// evaluateRotation marks a slot expiring once its connection exceeded the -// lifetime or bytes limit, and completes the rotation once the slot goes -// idle: the tired connection is closed so the next stream dials a fresh -// one. In-flight streams are never interrupted. -func (t *HTTP2Transport) evaluateRotation(idx int, s *transportSlot) { - if s.expiring.Load() { - if s.active.Load() == 0 { - s.t.CloseIdleConnections() - s.expiring.Store(false) - stats.RecordConnRotated() - log.Info("[TRANSPORT] connection rotated", "slot", idx) - } - return - } - if t.rotationDue(s, time.Now()) { - s.expiring.Store(true) - log.Info("[TRANSPORT] slot connection expiring", "slot", idx) - } -} - -// rotationDue reports whether the slot's connection exceeded the lifetime -// or bytes limit and should stop accepting new streams. -func (t *HTTP2Transport) rotationDue(s *transportSlot, now time.Time) bool { - if t.connLifetime > 0 { - if expireAt := s.expireAt.Load(); expireAt > 0 && now.UnixNano() >= expireAt { - return true - } - } - if t.connMaxBytes > 0 && s.connBytesRecv.Load() >= t.connMaxBytes { - return true - } - return false -} - -// rotationLifetime returns the connection lifetime with a per-connection -// random jitter of up to 20%. Connections created in the same burst (e.g. -// several slots dialing together) then expire in different health ticks, -// so rotations and the subsequent TLS handshakes do not cluster into a -// single fingerprintable burst. -func rotationLifetime(base time.Duration) time.Duration { - span := int64(base) / 5 - if span <= 0 { - return base - } - return base + time.Duration(rand.Int64N(span+1)) -} - -// retireSlot closes a degraded slot's idle connection and shrinks liveCount -// once the slot no longer hosts any stream. Streams are re-checked under -// the lock so a concurrent Open cannot be stranded. -func (t *HTTP2Transport) retireSlot(s *transportSlot) { - s.t.CloseIdleConnections() - t.mu.Lock() - defer t.mu.Unlock() - if s.active.Load() != 0 { - return - } - live := int(t.liveCount.Load()) - for i := 0; i < live; i++ { - if t.slots[i] != s { - continue - } - last := live - 1 - if i != last { - t.slots[i], t.slots[last] = t.slots[last], t.slots[i] - } - t.liveCount.Add(-1) - stats.RecordSlotRetiredDegraded() - log.Info("[TRANSPORT] slot retired (degraded)", "slot", i) - return - } + t.sched.mu.Lock() + defer t.sched.mu.Unlock() + t.sched.shrinkIdleLocked() } func (t *HTTP2Transport) Stats() transport.TransportStats { - live := int(t.liveCount.Load()) + live := int(t.sched.liveCount.Load()) ts := transport.TransportStats{ Conns: live, } - pConns := t.prioritySlots + pConns := t.sched.prioritySlots if live < pConns { pConns = live } @@ -698,9 +302,9 @@ func (t *HTTP2Transport) Stats() transport.TransportStats { ts.BulkConns = live - pConns for i := int32(0); i < int32(live); i++ { - a := int(t.slots[i].active.Load()) + a := int(t.sched.slots[i].active.Load()) ts.ActiveStreams += a - if i < int32(t.prioritySlots) { + if i < int32(t.sched.prioritySlots) { ts.PriorityActiveStreams += a } else { ts.BulkActiveStreams += a @@ -711,8 +315,8 @@ func (t *HTTP2Transport) Stats() transport.TransportStats { func (t *HTTP2Transport) Close() error { t.cancel() - live := t.liveCount.Load() - for _, s := range t.slots[:live] { + live := t.sched.liveCount.Load() + for _, s := range t.sched.slots[:live] { s.t.CloseIdleConnections() } return nil diff --git a/transport/http2/client_test.go b/transport/http2/client_test.go index 0bfde6fe..795dc18d 100644 --- a/transport/http2/client_test.go +++ b/transport/http2/client_test.go @@ -141,54 +141,6 @@ func TestHTTP2Transport_200StatusReadsBody(t *testing.T) { } } -// newTestSlots builds live slots with explicit active/heavy counters. -// Transport structs inside slots are left nil — scheduling tests never dial. -func newTestSlots(specs ...[2]int32) *HTTP2Transport { - slots := make([]*transportSlot, len(specs)) - for i, s := range specs { - slots[i] = &transportSlot{} - slots[i].active.Store(s[0]) - slots[i].heavy.Store(s[1]) - } - tr := &HTTP2Transport{ - slots: slots, - maxSlots: len(slots), - } - tr.liveCount.Store(int32(len(slots))) - return tr -} - -func TestLeastActiveSlotRangeSkipsHeavy(t *testing.T) { - t.Run("prefers non-heavy slot", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 1}, [2]int32{2, 0}) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[1] { - t.Fatalf("expected non-heavy slot 1, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) - } - }) - - t.Run("picks least active among non-heavy", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 0}, [2]int32{2, 1}, [2]int32{3, 0}) - if got := tr.leastActiveSlotRange(0, 3); got != tr.slots[2] { - t.Fatalf("expected least-active non-heavy slot 2, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) - } - }) - - t.Run("falls back to least active when all heavy", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 1}, [2]int32{2, 1}) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[1] { - t.Fatalf("expected least-active slot 1, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) - } - }) - - t.Run("respects liveCount bound", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 0}, [2]int32{2, 0}, [2]int32{3, 0}) - tr.liveCount.Store(2) - if got := tr.leastActiveSlotRange(0, 3); got != tr.slots[1] { - t.Fatalf("expected slot 1 within live range, got active=%d", got.active.Load()) - } - }) -} - func TestTrackReadMarksSlotHeavy(t *testing.T) { // Fast path: a large transfer is marked as soon as it crosses the // cumulative size threshold. @@ -292,314 +244,3 @@ func TestTrackReadMarksSlotHeavy(t *testing.T) { } }) } - -func TestEvaluateSlotHealth(t *testing.T) { - interval := sharedconfig.HealthCheckInterval - tr := &HTTP2Transport{} - - newHeavySlot := func() *transportSlot { - s := &transportSlot{t: &http.Transport{}} - s.heavy.Store(1) - return s - } - // lowThroughput simulates one health interval carrying 10KB - // (2KB/s, well below the 64KB/s degraded threshold). - lowThroughput := func(s *transportSlot) { - s.bytesRecv.Add(10 * 1024) - tr.evaluateSlotHealth(0, s, interval, true) - } - highThroughput := func(s *transportSlot) { - s.bytesRecv.Add(2 * 1024 * 1024) // 2MB over 5s = 400KB/s - tr.evaluateSlotHealth(0, s, interval, true) - } - - t.Run("marks degraded after consecutive slow intervals", func(t *testing.T) { - s := newHeavySlot() - // The first interval after heavy 0->1 only resets the throughput - // baseline and is skipped. - lowThroughput(s) - for i := 0; i < sharedconfig.DegradedPersistCycles-1; i++ { - lowThroughput(s) - if s.degraded.Load() { - t.Fatalf("degraded too early at cycle %d", i+1) - } - } - lowThroughput(s) - if !s.degraded.Load() { - t.Fatal("expected degraded after persist cycles") - } - }) - - t.Run("healthy interval resets the slow counter", func(t *testing.T) { - s := newHeavySlot() - lowThroughput(s) // baseline reset - lowThroughput(s) - lowThroughput(s) - highThroughput(s) - lowThroughput(s) - lowThroughput(s) - if s.degraded.Load() { - t.Fatal("expected not degraded after a healthy interval") - } - }) - - t.Run("clears degraded after consecutive healthy intervals", func(t *testing.T) { - s := newHeavySlot() - lowThroughput(s) // baseline reset - for i := 0; i < sharedconfig.DegradedPersistCycles; i++ { - lowThroughput(s) - } - if !s.degraded.Load() { - t.Fatal("expected degraded") - } - highThroughput(s) - if !s.degraded.Load() { - t.Fatal("cleared too early after a single healthy interval") - } - highThroughput(s) - if s.degraded.Load() { - t.Fatal("expected cleared after recover cycles") - } - }) - - t.Run("slots without heavy streams never degrade", func(t *testing.T) { - s := &transportSlot{t: &http.Transport{}} - for i := 0; i < sharedconfig.DegradedPersistCycles+2; i++ { - lowThroughput(s) - } - if s.degraded.Load() { - t.Fatal("non-heavy slot must not degrade") - } - }) - - t.Run("congested link never degrades", func(t *testing.T) { - s := newHeavySlot() - // Baseline reset happens before the congestion gate. - tr.evaluateSlotHealth(0, s, interval, false) - for i := 0; i < sharedconfig.DegradedPersistCycles+2; i++ { - s.bytesRecv.Add(10 * 1024) - tr.evaluateSlotHealth(0, s, interval, false) - } - if s.degraded.Load() { - t.Fatal("degraded while link congested") - } - }) - - t.Run("heavy 0->1 transition resets throughput baseline", func(t *testing.T) { - s := newHeavySlot() - s.bytesRecv.Add(50 * 1024) // stale bytes from earlier small streams - // First sample only resets the baseline. - tr.evaluateSlotHealth(0, s, interval, true) - if s.lowCycles != 0 { - t.Fatalf("lowCycles = %d after baseline reset, want 0", s.lowCycles) - } - // A slow interval afterwards counts from the reset point. - s.bytesRecv.Add(10 * 1024) - tr.evaluateSlotHealth(0, s, interval, true) - if s.lowCycles != 1 { - t.Fatalf("lowCycles = %d, want 1", s.lowCycles) - } - }) -} - -func TestRetireSlotShrinksLiveCount(t *testing.T) { - s0 := &transportSlot{t: &http.Transport{}} - s0.active.Store(1) - s1 := &transportSlot{t: &http.Transport{}} - s1.degraded.Store(true) - tr := &HTTP2Transport{slots: []*transportSlot{s0, s1}} - tr.liveCount.Store(2) - - tr.retireSlot(s1) - if got := tr.liveCount.Load(); got != 1 { - t.Fatalf("liveCount = %d, want 1", got) - } - if tr.slots[0] != s0 { - t.Fatal("live slot must stay at the front") - } - - // Busy slots are never retired. - s2 := &transportSlot{t: &http.Transport{}} - s2.active.Store(1) - s2.degraded.Store(true) - tr2 := &HTTP2Transport{slots: []*transportSlot{s2}} - tr2.liveCount.Store(1) - tr2.retireSlot(s2) - if got := tr2.liveCount.Load(); got != 1 { - t.Fatalf("busy slot retired: liveCount = %d", got) - } -} - -func TestLeastActiveSlotRangePrefersHealthyOverDegraded(t *testing.T) { - t.Run("skips degraded slot when healthy exists", func(t *testing.T) { - tr := newTestSlots([2]int32{2, 0}, [2]int32{1, 0}) - tr.slots[1].degraded.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[0] { - t.Fatalf("expected healthy slot 0, got active=%d heavy=%d degraded=%v", got.active.Load(), got.heavy.Load(), got.degraded.Load()) - } - }) - - t.Run("prefers heavy-but-not-degraded over degraded", func(t *testing.T) { - tr := newTestSlots([2]int32{1, 1}, [2]int32{3, 1}) - tr.slots[1].degraded.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[0] { - t.Fatalf("expected non-degraded slot 0, got active=%d degraded=%v", got.active.Load(), got.degraded.Load()) - } - }) - - t.Run("falls back to degraded when nothing else", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 1}, [2]int32{3, 1}) - tr.slots[0].degraded.Store(true) - tr.slots[1].degraded.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[1] { - t.Fatalf("expected least-active degraded slot 1, got active=%d", got.active.Load()) - } - }) -} - -func TestLeastActiveSlotRangePrefersNonExpiring(t *testing.T) { - t.Run("skips expiring slot when fresh exists", func(t *testing.T) { - tr := newTestSlots([2]int32{2, 0}, [2]int32{1, 0}) - tr.slots[1].expiring.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[0] { - t.Fatalf("expected non-expiring slot 0, got active=%d expiring=%v", got.active.Load(), got.expiring.Load()) - } - }) - - t.Run("prefers degraded-but-fresh over expiring", func(t *testing.T) { - tr := newTestSlots([2]int32{3, 1}, [2]int32{1, 1}) - tr.slots[0].degraded.Store(true) - tr.slots[1].expiring.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[0] { - t.Fatalf("expected degraded slot 0 over expiring, got active=%d", got.active.Load()) - } - }) - - t.Run("falls back to expiring when nothing else", func(t *testing.T) { - tr := newTestSlots([2]int32{5, 1}, [2]int32{3, 1}) - tr.slots[0].expiring.Store(true) - tr.slots[1].expiring.Store(true) - if got := tr.leastActiveSlotRange(0, 2); got != tr.slots[1] { - t.Fatalf("expected least-active expiring slot 1, got active=%d", got.active.Load()) - } - }) -} - -func TestShouldGrow(t *testing.T) { - t.Run("heavy slot with single stream does not block growth", func(t *testing.T) { - tr := newTestSlots([2]int32{1, 1}, [2]int32{8, 0}) - // slot0 is heavy with 1 stream (below threshold), slot1 eligible at - // threshold: growth must be allowed since new streams avoid slot0. - if !tr.shouldGrow(0, 2, 4) { - t.Fatal("expected growth with heavy slot below threshold") - } - }) - - t.Run("eligible slot below threshold blocks growth", func(t *testing.T) { - tr := newTestSlots([2]int32{3, 0}, [2]int32{8, 0}) - if tr.shouldGrow(0, 2, 4) { - t.Fatal("expected no growth while an eligible slot has capacity") - } - }) - - t.Run("all slots heavy still grows", func(t *testing.T) { - tr := newTestSlots([2]int32{1, 1}, [2]int32{1, 1}) - if !tr.shouldGrow(0, 2, 4) { - t.Fatal("expected growth when no eligible slot exists") - } - }) - - t.Run("degraded and expiring slots do not block growth", func(t *testing.T) { - tr := newTestSlots([2]int32{1, 0}, [2]int32{8, 0}) - tr.slots[0].degraded.Store(true) - if !tr.shouldGrow(0, 2, 4) { - t.Fatal("expected growth with degraded slot below threshold") - } - tr2 := newTestSlots([2]int32{1, 0}, [2]int32{8, 0}) - tr2.slots[0].expiring.Store(true) - if !tr2.shouldGrow(0, 2, 4) { - t.Fatal("expected growth with expiring slot below threshold") - } - }) -} - -func TestRotationDue(t *testing.T) { - now := time.Now() - t.Run("lifetime exceeded", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Minute} - s := &transportSlot{} - s.expireAt.Store(now.Add(-2 * time.Minute).UnixNano()) - if !tr.rotationDue(s, now) { - t.Fatal("expected rotation due by age") - } - }) - - t.Run("bytes exceeded", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Hour, connMaxBytes: 1024} - s := &transportSlot{} - s.expireAt.Store(now.Add(time.Hour).UnixNano()) - s.connBytesRecv.Store(2048) - if !tr.rotationDue(s, now) { - t.Fatal("expected rotation due by bytes") - } - }) - - t.Run("fresh connection not due", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Minute, connMaxBytes: 1024} - s := &transportSlot{} - s.expireAt.Store(now.Add(time.Minute).UnixNano()) - s.connBytesRecv.Store(512) - if tr.rotationDue(s, now) { - t.Fatal("fresh connection must not rotate") - } - }) - - t.Run("never dialed slot not due by age", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Minute} - s := &transportSlot{} - if tr.rotationDue(s, now) { - t.Fatal("undialed slot must not rotate") - } - }) -} - -func TestRotationLifetimeJitter(t *testing.T) { - const base = 15 * time.Minute - max := base + base/5 - for i := 0; i < 1000; i++ { - got := rotationLifetime(base) - if got < base || got > max { - t.Fatalf("rotationLifetime(%v) = %v, want within [%v, %v]", base, got, base, max) - } - } -} - -func TestEvaluateRotation(t *testing.T) { - t.Run("marks expiring and completes rotation when idle", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Minute} - s := &transportSlot{t: &http.Transport{}} - s.expireAt.Store(time.Now().Add(-2 * time.Minute).UnixNano()) - s.active.Store(1) - // First pass: still busy, only mark expiring. - tr.evaluateRotation(0, s) - if !s.expiring.Load() { - t.Fatal("expected expiring mark") - } - // Second pass: idle now, rotation completes and clears the mark. - s.active.Store(0) - tr.evaluateRotation(0, s) - if s.expiring.Load() { - t.Fatal("expected expiring cleared after rotation") - } - }) - - t.Run("fresh connection not expiring", func(t *testing.T) { - tr := &HTTP2Transport{connLifetime: time.Hour} - s := &transportSlot{t: &http.Transport{}} - s.expireAt.Store(time.Now().Add(time.Hour).UnixNano()) - tr.evaluateRotation(0, s) - if s.expiring.Load() { - t.Fatal("fresh connection must not expire") - } - }) -} diff --git a/transport/http2/lifecycle.go b/transport/http2/lifecycle.go new file mode 100644 index 00000000..ecae43d7 --- /dev/null +++ b/transport/http2/lifecycle.go @@ -0,0 +1,180 @@ +package http2 + +import ( + "context" + "math/rand/v2" + "time" + + sharedconfig "github.com/nange/easyss/v3/config" + "github.com/nange/easyss/v3/log" + "github.com/nange/easyss/v3/stats" +) + +// slotLifecycle owns the per-slot connection lifecycle: a health loop that +// samples download throughput for degraded detection, and connection +// rotation once the lifetime or bytes limit is exceeded. It reuses the +// scheduler's pool management to retire idle degraded slots. +type slotLifecycle struct { + sched *slotScheduler + connLifetime time.Duration // max age of a connection before rotation + connMaxBytes int64 // max bytes per connection before rotation +} + +// run drives the periodic health evaluation until ctx is cancelled. +func (lc *slotLifecycle) run(ctx context.Context) { + interval := sharedconfig.HealthCheckInterval + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + lc.evaluate(interval) + case <-ctx.Done(): + return + } + } +} + +// evaluate walks the live slots: download throughput feeds the degraded +// detector, connection age/bytes feed rotation, and idle degraded slots are +// retired. +func (lc *slotLifecycle) evaluate(interval time.Duration) { + // A congested link (high RTT) makes every connection slow: marking or + // retiring slots then only adds handshake churn without recovering + // anything, so degraded detection is gated on a healthy RTT. Rotation, + // on the other hand, is exactly what a throttled connection needs and + // runs regardless of RTT. + linkOK := stats.Collect().AvgRTT() <= sharedconfig.DegradedMaxRTT + + live := int(lc.sched.liveCount.Load()) + for i := 0; i < live; i++ { + s := lc.sched.slots[i] + lc.evaluateSlotHealth(i, s, interval, linkOK) + lc.evaluateRotation(i, s) + if linkOK && s.degraded.Load() && s.active.Load() == 0 { + lc.retire(i, s) + // retire swap-removes the slot to the end and shrinks + // liveCount; re-evaluate the slot swapped into position i. + live-- + i-- + } + } +} + +// evaluateSlotHealth updates one slot's degraded state from its recent +// download throughput. Only slots hosting heavy streams are considered — +// idle or short-lived slots naturally carry zero throughput. The mark is +// set after DegradedPersistCycles consecutive slow intervals and cleared +// after DegradedRecoverCycles healthy ones. +func (lc *slotLifecycle) evaluateSlotHealth(idx int, s *transportSlot, interval time.Duration, linkOK bool) { + if s.heavy.Load() == 0 { + s.lastHeavy = 0 + return + } + if s.lastHeavy == 0 { + // First heavy stream on this slot since it went idle: bytes + // transferred by earlier small streams must not skew the first + // sample, so reset the baseline and skip this interval. + s.lastHeavy = 1 + s.lastBytes = s.bytesRecv.Load() + s.lowCycles = 0 + s.recoverCycles = 0 + return + } + if !linkOK { + // Congested link: low throughput is a link property, not evidence + // that this particular connection is broken. Advance the baseline + // so the first healthy interval starts from a clean sample, and + // freeze the counters. + s.lastBytes = s.bytesRecv.Load() + return + } + + now := s.bytesRecv.Load() + perSec := int64(interval / time.Second) + if perSec <= 0 { + perSec = 1 + } + throughput := (now - s.lastBytes) / perSec + s.lastBytes = now + + if throughput >= int64(sharedconfig.DegradedThroughputThreshold) { + s.lowCycles = 0 + if s.degraded.Load() { + s.recoverCycles++ + if s.recoverCycles >= sharedconfig.DegradedRecoverCycles { + s.degraded.Store(false) + s.recoverCycles = 0 + log.Info("[TRANSPORT] slot recovered", "slot", idx, "throughput_kb_s", throughput/1024) + } + } + return + } + + s.recoverCycles = 0 + s.lowCycles++ + if s.lowCycles >= sharedconfig.DegradedPersistCycles && !s.degraded.Load() { + s.degraded.Store(true) + s.lowCycles = 0 + stats.RecordSlotDegraded() + log.Info("[TRANSPORT] slot degraded", "slot", idx, "throughput_kb_s", throughput/1024) + } +} + +// evaluateRotation marks a slot expiring once its connection exceeded the +// lifetime or bytes limit, and completes the rotation once the slot goes +// idle: the tired connection is closed so the next stream dials a fresh +// one. In-flight streams are never interrupted. +func (lc *slotLifecycle) evaluateRotation(idx int, s *transportSlot) { + if s.expiring.Load() { + if s.active.Load() == 0 { + s.t.CloseIdleConnections() + s.expiring.Store(false) + stats.RecordConnRotated() + log.Info("[TRANSPORT] connection rotated", "slot", idx) + } + return + } + if lc.rotationDue(s, time.Now()) { + s.expiring.Store(true) + log.Info("[TRANSPORT] slot connection expiring", "slot", idx) + } +} + +// rotationDue reports whether the slot's connection exceeded the lifetime +// or bytes limit and should stop accepting new streams. +func (lc *slotLifecycle) rotationDue(s *transportSlot, now time.Time) bool { + if lc.connLifetime > 0 { + if expireAt := s.expireAt.Load(); expireAt > 0 && now.UnixNano() >= expireAt { + return true + } + } + if lc.connMaxBytes > 0 && s.connBytesRecv.Load() >= lc.connMaxBytes { + return true + } + return false +} + +// rotationLifetime returns the connection lifetime with a per-connection +// random jitter of up to 20%. Connections created in the same burst (e.g. +// several slots dialing together) then expire in different health ticks, +// so rotations and the subsequent TLS handshakes do not cluster into a +// single fingerprintable burst. +func rotationLifetime(base time.Duration) time.Duration { + span := int64(base) / 5 + if span <= 0 { + return base + } + return base + time.Duration(rand.Int64N(span+1)) +} + +// retire closes a degraded slot's idle connection and removes the slot from +// the live set once it no longer hosts any stream. +func (lc *slotLifecycle) retire(idx int, s *transportSlot) { + s.t.CloseIdleConnections() + if !lc.sched.remove(s) { + return + } + stats.RecordSlotRetiredDegraded() + log.Info("[TRANSPORT] slot retired (degraded)", "slot", idx) +} diff --git a/transport/http2/lifecycle_test.go b/transport/http2/lifecycle_test.go new file mode 100644 index 00000000..bb3ec171 --- /dev/null +++ b/transport/http2/lifecycle_test.go @@ -0,0 +1,199 @@ +package http2 + +import ( + "net/http" + "testing" + "time" + + sharedconfig "github.com/nange/easyss/v3/config" +) + +func TestEvaluateSlotHealth(t *testing.T) { + interval := sharedconfig.HealthCheckInterval + lc := &slotLifecycle{} + + newHeavySlot := func() *transportSlot { + s := &transportSlot{t: &http.Transport{}} + s.heavy.Store(1) + return s + } + // lowThroughput simulates one health interval carrying 10KB + // (2KB/s, well below the 64KB/s degraded threshold). + lowThroughput := func(s *transportSlot) { + s.bytesRecv.Add(10 * 1024) + lc.evaluateSlotHealth(0, s, interval, true) + } + highThroughput := func(s *transportSlot) { + s.bytesRecv.Add(2 * 1024 * 1024) // 2MB over 5s = 400KB/s + lc.evaluateSlotHealth(0, s, interval, true) + } + + t.Run("marks degraded after consecutive slow intervals", func(t *testing.T) { + s := newHeavySlot() + // The first interval after heavy 0->1 only resets the throughput + // baseline and is skipped. + lowThroughput(s) + for i := 0; i < sharedconfig.DegradedPersistCycles-1; i++ { + lowThroughput(s) + if s.degraded.Load() { + t.Fatalf("degraded too early at cycle %d", i+1) + } + } + lowThroughput(s) + if !s.degraded.Load() { + t.Fatal("expected degraded after persist cycles") + } + }) + + t.Run("healthy interval resets the slow counter", func(t *testing.T) { + s := newHeavySlot() + lowThroughput(s) // baseline reset + lowThroughput(s) + lowThroughput(s) + highThroughput(s) + lowThroughput(s) + lowThroughput(s) + if s.degraded.Load() { + t.Fatal("expected not degraded after a healthy interval") + } + }) + + t.Run("clears degraded after consecutive healthy intervals", func(t *testing.T) { + s := newHeavySlot() + lowThroughput(s) // baseline reset + for i := 0; i < sharedconfig.DegradedPersistCycles; i++ { + lowThroughput(s) + } + if !s.degraded.Load() { + t.Fatal("expected degraded") + } + highThroughput(s) + if !s.degraded.Load() { + t.Fatal("cleared too early after a single healthy interval") + } + highThroughput(s) + if s.degraded.Load() { + t.Fatal("expected cleared after recover cycles") + } + }) + + t.Run("slots without heavy streams never degrade", func(t *testing.T) { + s := &transportSlot{t: &http.Transport{}} + for i := 0; i < sharedconfig.DegradedPersistCycles+2; i++ { + lowThroughput(s) + } + if s.degraded.Load() { + t.Fatal("non-heavy slot must not degrade") + } + }) + + t.Run("congested link never degrades", func(t *testing.T) { + s := newHeavySlot() + // Baseline reset happens before the congestion gate. + lc.evaluateSlotHealth(0, s, interval, false) + for i := 0; i < sharedconfig.DegradedPersistCycles+2; i++ { + s.bytesRecv.Add(10 * 1024) + lc.evaluateSlotHealth(0, s, interval, false) + } + if s.degraded.Load() { + t.Fatal("degraded while link congested") + } + }) + + t.Run("heavy 0->1 transition resets throughput baseline", func(t *testing.T) { + s := newHeavySlot() + s.bytesRecv.Add(50 * 1024) // stale bytes from earlier small streams + // First sample only resets the baseline. + lc.evaluateSlotHealth(0, s, interval, true) + if s.lowCycles != 0 { + t.Fatalf("lowCycles = %d after baseline reset, want 0", s.lowCycles) + } + // A slow interval afterwards counts from the reset point. + s.bytesRecv.Add(10 * 1024) + lc.evaluateSlotHealth(0, s, interval, true) + if s.lowCycles != 1 { + t.Fatalf("lowCycles = %d, want 1", s.lowCycles) + } + }) +} + +func TestRotationDue(t *testing.T) { + now := time.Now() + t.Run("lifetime exceeded", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Minute} + s := &transportSlot{} + s.expireAt.Store(now.Add(-2 * time.Minute).UnixNano()) + if !lc.rotationDue(s, now) { + t.Fatal("expected rotation due by age") + } + }) + + t.Run("bytes exceeded", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Hour, connMaxBytes: 1024} + s := &transportSlot{} + s.expireAt.Store(now.Add(time.Hour).UnixNano()) + s.connBytesRecv.Store(2048) + if !lc.rotationDue(s, now) { + t.Fatal("expected rotation due by bytes") + } + }) + + t.Run("fresh connection not due", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Minute, connMaxBytes: 1024} + s := &transportSlot{} + s.expireAt.Store(now.Add(time.Minute).UnixNano()) + s.connBytesRecv.Store(512) + if lc.rotationDue(s, now) { + t.Fatal("fresh connection must not rotate") + } + }) + + t.Run("never dialed slot not due by age", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Minute} + s := &transportSlot{} + if lc.rotationDue(s, now) { + t.Fatal("undialed slot must not rotate") + } + }) +} + +func TestRotationLifetimeJitter(t *testing.T) { + const base = 15 * time.Minute + max := base + base/5 + for i := 0; i < 1000; i++ { + got := rotationLifetime(base) + if got < base || got > max { + t.Fatalf("rotationLifetime(%v) = %v, want within [%v, %v]", base, got, base, max) + } + } +} + +func TestEvaluateRotation(t *testing.T) { + t.Run("marks expiring and completes rotation when idle", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Minute} + s := &transportSlot{t: &http.Transport{}} + s.expireAt.Store(time.Now().Add(-2 * time.Minute).UnixNano()) + s.active.Store(1) + // First pass: still busy, only mark expiring. + lc.evaluateRotation(0, s) + if !s.expiring.Load() { + t.Fatal("expected expiring mark") + } + // Second pass: idle now, rotation completes and clears the mark. + s.active.Store(0) + lc.evaluateRotation(0, s) + if s.expiring.Load() { + t.Fatal("expected expiring cleared after rotation") + } + }) + + t.Run("fresh connection not expiring", func(t *testing.T) { + lc := &slotLifecycle{connLifetime: time.Hour} + s := &transportSlot{t: &http.Transport{}} + s.expireAt.Store(time.Now().Add(time.Hour).UnixNano()) + lc.evaluateRotation(0, s) + if s.expiring.Load() { + t.Fatal("fresh connection must not expire") + } + }) +} diff --git a/transport/http2/scheduler.go b/transport/http2/scheduler.go new file mode 100644 index 00000000..3b31df90 --- /dev/null +++ b/transport/http2/scheduler.go @@ -0,0 +1,246 @@ +package http2 + +import ( + "math" + "sync" + "sync/atomic" + + "github.com/nange/easyss/v3/stats" +) + +// slotScheduler maps new streams onto slots (least-active, health-aware) +// and manages the live slot set: growth under load, shrink on idle, and +// swap-remove of retired slots. Slot health state itself is owned by +// slotLifecycle; the scheduler only consumes the eligibility signals. +type slotScheduler struct { + slots []*transportSlot // pre-allocated and initialized to maxSlots + liveCount atomic.Int32 // number of currently live slots (0..maxSlots) + maxSlots int + threshold int32 + prioritySlots int // number of priority slots (0..prioritySlots-1) + bulkThreshold int32 + mu sync.RWMutex // protects slot retire (shrink) and grow; RLock protects stream assignment +} + +func newScheduler(maxSlots int, slots []*transportSlot, threshold int32, prioritySlots int) *slotScheduler { + return &slotScheduler{ + slots: slots, + maxSlots: maxSlots, + threshold: threshold, + prioritySlots: prioritySlots, + bulkThreshold: threshold * 2, + } +} + +// pick returns the slot a new stream should use: priority streams prefer +// priority slots, bulk streams prefer the bulk range, each falling back to +// the other range when its own is exhausted. +func (s *slotScheduler) pick(highPriority bool) *transportSlot { + if highPriority && s.prioritySlots > 0 { + slot := s.leastActiveInRange(0, s.prioritySlots) + if slot == nil || slot.active.Load() >= s.threshold { + stats.RecordPriorityFallback() + slot = s.leastActiveInRange(s.prioritySlots, int(s.liveCount.Load())) + } + return slot + } + + slot := s.leastActiveInRange(s.prioritySlots, int(s.liveCount.Load())) + if slot == nil || slot.active.Load() >= s.bulkThreshold { + stats.RecordBulkFallback() + slot = s.leastActiveInRange(0, s.prioritySlots) + } + return slot +} + +// leastActiveInRange returns the slot in [start,end) with the fewest active +// streams, preferring healthy slots, then slots without heavy streams, and +// only falling back to degraded or expiring ones when nothing else is +// available: +// - a heavy stream monopolizes the connection's TCP window, so a new +// stream sharing that slot is dragged down together with it (TCP +// head-of-line blocking under packet loss); +// - a degraded slot already proved persistently low throughput, so it +// is avoided unless it is the only option left; +// - an expiring slot is due for connection rotation, so new streams +// avoid it unless nothing else is available. +func (s *slotScheduler) leastActiveInRange(start, end int) *transportSlot { + live := int(s.liveCount.Load()) + if live == 0 { + return s.slots[0] + } + if end > live { + end = live + } + if start >= end { + start = 0 + end = live + } + passes := []struct { + skipHeavy bool + skipDegraded bool + skipExpiring bool + }{ + {true, true, true}, // healthy slots + {false, true, true}, // heavy but not degraded/expiring + {false, false, true}, // degraded but not expiring + {false, false, false}, + } + var best *transportSlot + for _, p := range passes { + var min int32 = math.MaxInt32 + for i := start; i < end; i++ { + sl := s.slots[i] + if !sl.eligible(p.skipHeavy, p.skipDegraded, p.skipExpiring) { + continue + } + if a := sl.active.Load(); a < min { + best, min = sl, a + } + } + if best != nil { + return best + } + } + return s.slots[0] +} + +// grow activates one more live slot (up to maxSlots) when every eligible +// slot that a new stream of this class would use is at or above the +// threshold. Uses double-checked locking. +func (s *slotScheduler) grow(highPriority bool) { + live := s.liveCount.Load() + if int(live) >= s.maxSlots { + return + } + + thresh := s.threshold + start, end := int32(0), live + if highPriority && s.prioritySlots > 0 { + end = int32(s.prioritySlots) + if end > live { + end = live + } + } else if s.prioritySlots > 0 { + start = int32(s.prioritySlots) + thresh = s.bulkThreshold + } + + if live > 0 { + if start >= end { + return + } + if !s.needsMore(start, end, thresh) { + return + } + } + + // All eligible slots in range are at or above threshold — try to grow + // under lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Double-check after acquiring the lock. + live = s.liveCount.Load() + if int(live) >= s.maxSlots { + return + } + start2, end2 := int32(0), live + if highPriority && s.prioritySlots > 0 { + end2 = int32(s.prioritySlots) + if end2 > live { + end2 = live + } + } else if s.prioritySlots > 0 { + start2 = int32(s.prioritySlots) + } + if live > 0 { + if start2 >= end2 { + return + } + if !s.needsMore(start2, end2, thresh) { + return + } + } + + // On first activation, start with 2 connections for better initial throughput, + // since typical web browsing generates >8 concurrent streams. + // Falls back to 1 when maxSlots is 1. + if live == 0 && s.maxSlots >= 2 { + s.liveCount.Add(2) + } else { + s.liveCount.Add(1) + } +} + +// needsMore reports whether the range [start,end) needs one more live slot: +// every slot a new stream would actually use is at or above the threshold. +// Heavy, degraded and expiring slots are skipped — a new stream avoids +// them, so a heavy slot hosting a single download must not block growing +// more connections. A range with no eligible slot at all also grows, since +// new streams then fall back onto heavy/degraded slots and deserve a fresh +// connection. +func (s *slotScheduler) needsMore(start, end, thresh int32) bool { + for i := start; i < end; i++ { + sl := s.slots[i] + if !sl.eligible(true, true, true) { + continue + } + if sl.active.Load() < thresh { + return false + } + } + return true +} + +// shrinkIdleLocked retires every idle slot (active==0) from liveCount, +// swap-removing each to the end. Caller must hold s.mu. +func (s *slotScheduler) shrinkIdleLocked() { + for s.removeIdleLocked() { + } +} + +// removeIdleLocked swap-removes the first idle slot from liveCount. +// Returns false when no idle slot remains. Caller must hold s.mu. +func (s *slotScheduler) removeIdleLocked() bool { + live := int(s.liveCount.Load()) + for i := 0; i < live; i++ { + if s.slots[i].active.Load() != 0 { + continue + } + s.removeAtLocked(i, live) + return true + } + return false +} + +// removeAtLocked swap-removes the slot at position i from liveCount. +// Caller must hold s.mu. +func (s *slotScheduler) removeAtLocked(i, live int) { + last := live - 1 + if i != last { + s.slots[i], s.slots[last] = s.slots[last], s.slots[i] + } + s.liveCount.Add(-1) +} + +// remove swaps the slot out of liveCount (swap-remove) if it is still live +// and not hosting streams, and reports whether the removal happened. +// Streams are re-checked under the lock so a concurrent Open cannot be +// stranded. +func (s *slotScheduler) remove(sl *transportSlot) bool { + s.mu.Lock() + defer s.mu.Unlock() + if sl.active.Load() != 0 { + return false + } + live := int(s.liveCount.Load()) + for i := 0; i < live; i++ { + if s.slots[i] != sl { + continue + } + s.removeAtLocked(i, live) + return true + } + return false +} diff --git a/transport/http2/scheduler_test.go b/transport/http2/scheduler_test.go new file mode 100644 index 00000000..3b9d1fe9 --- /dev/null +++ b/transport/http2/scheduler_test.go @@ -0,0 +1,177 @@ +package http2 + +import ( + "net/http" + "testing" +) + +// newTestScheduler builds a live scheduler with explicit active/heavy +// counters. Transport structs inside slots are left nil — scheduling tests +// never dial. +func newTestScheduler(specs ...[2]int32) *slotScheduler { + slots := make([]*transportSlot, len(specs)) + for i, s := range specs { + slots[i] = &transportSlot{} + slots[i].active.Store(s[0]) + slots[i].heavy.Store(s[1]) + } + sch := newScheduler(len(slots), slots, 4, 1) + sch.liveCount.Store(int32(len(slots))) + return sch +} + +func TestLeastActiveInRangeSkipsHeavy(t *testing.T) { + t.Run("prefers non-heavy slot", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 1}, [2]int32{2, 0}) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[1] { + t.Fatalf("expected non-heavy slot 1, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) + } + }) + + t.Run("picks least active among non-heavy", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 0}, [2]int32{2, 1}, [2]int32{3, 0}) + if got := sch.leastActiveInRange(0, 3); got != sch.slots[2] { + t.Fatalf("expected least-active non-heavy slot 2, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) + } + }) + + t.Run("falls back to least active when all heavy", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 1}, [2]int32{2, 1}) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[1] { + t.Fatalf("expected least-active slot 1, got active=%d heavy=%d", got.active.Load(), got.heavy.Load()) + } + }) + + t.Run("respects liveCount bound", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 0}, [2]int32{2, 0}, [2]int32{3, 0}) + sch.liveCount.Store(2) + if got := sch.leastActiveInRange(0, 3); got != sch.slots[1] { + t.Fatalf("expected slot 1 within live range, got active=%d", got.active.Load()) + } + }) +} + +func TestLeastActiveInRangePrefersHealthyOverDegraded(t *testing.T) { + t.Run("skips degraded slot when healthy exists", func(t *testing.T) { + sch := newTestScheduler([2]int32{2, 0}, [2]int32{1, 0}) + sch.slots[1].degraded.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[0] { + t.Fatalf("expected healthy slot 0, got active=%d heavy=%d degraded=%v", got.active.Load(), got.heavy.Load(), got.degraded.Load()) + } + }) + + t.Run("prefers heavy-but-not-degraded over degraded", func(t *testing.T) { + sch := newTestScheduler([2]int32{1, 1}, [2]int32{3, 1}) + sch.slots[1].degraded.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[0] { + t.Fatalf("expected non-degraded slot 0, got active=%d degraded=%v", got.active.Load(), got.degraded.Load()) + } + }) + + t.Run("falls back to degraded when nothing else", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 1}, [2]int32{3, 1}) + sch.slots[0].degraded.Store(true) + sch.slots[1].degraded.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[1] { + t.Fatalf("expected least-active degraded slot 1, got active=%d", got.active.Load()) + } + }) +} + +func TestLeastActiveInRangePrefersNonExpiring(t *testing.T) { + t.Run("skips expiring slot when fresh exists", func(t *testing.T) { + sch := newTestScheduler([2]int32{2, 0}, [2]int32{1, 0}) + sch.slots[1].expiring.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[0] { + t.Fatalf("expected non-expiring slot 0, got active=%d expiring=%v", got.active.Load(), got.expiring.Load()) + } + }) + + t.Run("prefers degraded-but-fresh over expiring", func(t *testing.T) { + sch := newTestScheduler([2]int32{3, 1}, [2]int32{1, 1}) + sch.slots[0].degraded.Store(true) + sch.slots[1].expiring.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[0] { + t.Fatalf("expected degraded slot 0 over expiring, got active=%d", got.active.Load()) + } + }) + + t.Run("falls back to expiring when nothing else", func(t *testing.T) { + sch := newTestScheduler([2]int32{5, 1}, [2]int32{3, 1}) + sch.slots[0].expiring.Store(true) + sch.slots[1].expiring.Store(true) + if got := sch.leastActiveInRange(0, 2); got != sch.slots[1] { + t.Fatalf("expected least-active expiring slot 1, got active=%d", got.active.Load()) + } + }) +} + +func TestNeedsMore(t *testing.T) { + t.Run("heavy slot with single stream does not block growth", func(t *testing.T) { + sch := newTestScheduler([2]int32{1, 1}, [2]int32{8, 0}) + // slot0 is heavy with 1 stream (below threshold), slot1 eligible at + // threshold: growth must be allowed since new streams avoid slot0. + if !sch.needsMore(0, 2, 4) { + t.Fatal("expected growth with heavy slot below threshold") + } + }) + + t.Run("eligible slot below threshold blocks growth", func(t *testing.T) { + sch := newTestScheduler([2]int32{3, 0}, [2]int32{8, 0}) + if sch.needsMore(0, 2, 4) { + t.Fatal("expected no growth while an eligible slot has capacity") + } + }) + + t.Run("all slots heavy still grows", func(t *testing.T) { + sch := newTestScheduler([2]int32{1, 1}, [2]int32{1, 1}) + if !sch.needsMore(0, 2, 4) { + t.Fatal("expected growth when no eligible slot exists") + } + }) + + t.Run("degraded and expiring slots do not block growth", func(t *testing.T) { + sch := newTestScheduler([2]int32{1, 0}, [2]int32{8, 0}) + sch.slots[0].degraded.Store(true) + if !sch.needsMore(0, 2, 4) { + t.Fatal("expected growth with degraded slot below threshold") + } + sch2 := newTestScheduler([2]int32{1, 0}, [2]int32{8, 0}) + sch2.slots[0].expiring.Store(true) + if !sch2.needsMore(0, 2, 4) { + t.Fatal("expected growth with expiring slot below threshold") + } + }) +} + +func TestRemoveShrinksLiveCount(t *testing.T) { + s0 := &transportSlot{t: &http.Transport{}} + s0.active.Store(1) + s1 := &transportSlot{t: &http.Transport{}} + s1.degraded.Store(true) + sch := newScheduler(2, []*transportSlot{s0, s1}, 4, 1) + sch.liveCount.Store(2) + + if !sch.remove(s1) { + t.Fatal("idle slot must be removable") + } + if got := sch.liveCount.Load(); got != 1 { + t.Fatalf("liveCount = %d, want 1", got) + } + if sch.slots[0] != s0 { + t.Fatal("live slot must stay at the front") + } + + // Busy slots are never removed. + s2 := &transportSlot{t: &http.Transport{}} + s2.active.Store(1) + s2.degraded.Store(true) + sch2 := newScheduler(1, []*transportSlot{s2}, 4, 1) + sch2.liveCount.Store(1) + if sch2.remove(s2) { + t.Fatal("busy slot must not be removable") + } + if got := sch2.liveCount.Load(); got != 1 { + t.Fatalf("busy slot removed: liveCount = %d", got) + } +} diff --git a/transport/http2/slot.go b/transport/http2/slot.go new file mode 100644 index 00000000..c5e7f3d3 --- /dev/null +++ b/transport/http2/slot.go @@ -0,0 +1,63 @@ +package http2 + +import ( + "net/http" + "sync/atomic" + "time" +) + +// transportSlot hosts one HTTP/2 connection (a stdlib http.Transport pinned +// to a single connection via MaxConnsPerHost=1) plus the state that the +// scheduler and the connection lifecycle act on. +type transportSlot struct { + t *http.Transport + active atomic.Int32 + heavy atomic.Int32 // number of active heavy streams (>= HeavyStreamThreshold bytes) + // bytesRecv is the cumulative downloaded bytes across streams on this + // slot; sampled by the health loop to estimate recent throughput. + bytesRecv atomic.Int64 + // degraded marks a slot whose download throughput stays below the + // degraded threshold while hosting heavy streams; new streams avoid it + // and its idle connection is retired early. + degraded atomic.Bool + + // Connection rotation state. + expireAt atomic.Int64 // unix nano deadline of the current connection (dial time + lifetime + jitter) + connBytesRecv atomic.Int64 // bytes downloaded over the current connection + // expiring marks a slot whose connection exceeded the lifetime or bytes + // limit; new streams avoid it and its idle connection is closed so the + // next stream dials a fresh one. Cleared when a new connection is + // established or the rotation completes. + expiring atomic.Bool + + // Health-loop state, touched only from the health loop goroutine. + lastBytes int64 + lastHeavy int // last observed heavy count, tracks heavy 0->1 transitions + lowCycles int + recoverCycles int +} + +// eligible reports whether the slot may host a new stream under the given +// skip filters. The scheduler queries it instead of reading the flags +// directly, so the meaning of "healthy" lives with the slot. +func (s *transportSlot) eligible(skipHeavy, skipDegraded, skipExpiring bool) bool { + if skipHeavy && s.heavy.Load() > 0 { + return false + } + if skipDegraded && s.degraded.Load() { + return false + } + if skipExpiring && s.expiring.Load() { + return false + } + return true +} + +// resetConn (re)initializes the rotation state for a freshly established +// connection: the lifetime deadline (with per-connection jitter), the bytes +// carried and the expiring mark all start fresh. +func (s *transportSlot) resetConn(connLifetime time.Duration) { + s.expireAt.Store(time.Now().Add(rotationLifetime(connLifetime)).UnixNano()) + s.connBytesRecv.Store(0) + s.expiring.Store(false) +}