diff --git a/cmd/skywire-cli/commands/proxy/mux_ops.go b/cmd/skywire-cli/commands/proxy/mux_ops.go index cfff2c0852..4ddba70d9a 100644 --- a/cmd/skywire-cli/commands/proxy/mux_ops.go +++ b/cmd/skywire-cli/commands/proxy/mux_ops.go @@ -37,6 +37,8 @@ import ( "io" "os" + "strconv" + "github.com/google/uuid" "github.com/spf13/cobra" @@ -62,6 +64,66 @@ func init() { addMuxSub(muxAddCmd, "mux-add") addMuxSub(muxRmCmd, "mux-rm") addMuxSub(muxModeCmd, "mux-mode") + addMuxSub(muxCapCmd, "mux-cap") + addMuxSub(muxWidthCmd, "mux-width") +} + +var muxCapCmd = &cobra.Command{ + Use: "cap ", + Short: "Set the adaptive mux active-width ceiling at runtime", + Long: `Set the MAXIMUM number of ACTIVE mux legs the adaptive engine may grow to +under sustained load — the aggregation ceiling. Applies LIVE to this visor's +adaptive route groups on their next tick (no restart). Send-side is a per-visor +decision, so set it independently on each end (e.g. over the pty to the exit). + +Example: + skywire cli proxy mux cap 60 # allow aggregation up to 60 active legs`, + Args: cobra.ExactArgs(1), + DisableFlagsInUseLine: true, + Run: func(cmd *cobra.Command, args []string) { + n, err := strconv.Atoi(args[0]) + if err != nil || n < 1 { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("cap must be a positive integer, got %q", args[0])) + } + rpcClient, err := clirpc.Client(cmd.Flags()) + if err != nil { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("unable to create RPC client: %w", err)) + } + defer rpcClient.Close() //nolint:errcheck,gosec + if err := rpcClient.SetMuxCap(n); err != nil { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("SetMuxCap: %w", err)) + } + internal.Catch(cmd.Flags(), cliout.Print(cmd, cliproxy.MuxOp{Op: "cap", App: muxOpsApp, Mode: args[0]})) + }, +} + +var muxWidthCmd = &cobra.Command{ + Use: "width ", + Short: "Set the adaptive mux steady active download width at runtime", + Long: `Set the STEADY active download width — the floor number of active mux legs +the adaptive engine converges to when idle (more than one spreads a bulk flow +proactively before saturation instead of ramping from a single leg). Applies +LIVE on the next tick; clamped to [1, cap]. Set per-visor, per-end. + +Example: + skywire cli proxy mux width 8 # keep 8 legs active by default`, + Args: cobra.ExactArgs(1), + DisableFlagsInUseLine: true, + Run: func(cmd *cobra.Command, args []string) { + n, err := strconv.Atoi(args[0]) + if err != nil || n < 1 { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("width must be a positive integer, got %q", args[0])) + } + rpcClient, err := clirpc.Client(cmd.Flags()) + if err != nil { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("unable to create RPC client: %w", err)) + } + defer rpcClient.Close() //nolint:errcheck,gosec + if err := rpcClient.SetMuxWidth(n); err != nil { + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("SetMuxWidth: %w", err)) + } + internal.Catch(cmd.Flags(), cliout.Print(cmd, cliproxy.MuxOp{Op: "width", App: muxOpsApp, Mode: args[0]})) + }, } // routePair mirrors the shape 'cli route calc --json' emits. @@ -195,30 +257,38 @@ Example: } var muxModeCmd = &cobra.Command{ - Use: "mode ", + Use: "mode ", Short: "Change mux scheduler weighting at runtime", Long: `Set the mux transport-selection mode for the visor. - auto - latency-weighted: lower-latency legs get more packets. - Best when the legs have different RTTs (the typical case) - because it minimizes head-of-line stalls in SACK reorder. - equal - round-robin: each leg gets equal share. Useful when legs - have similar latency and you want to verify aggregation - behavior without the auto-mode masking it. + auto - latency-weighted: lower-latency legs get more packets. + Best when the legs have different RTTs (the typical case) + because it minimizes head-of-line stalls in SACK reorder. + equal - round-robin: each leg gets equal share. Useful when legs + have similar latency and you want to verify aggregation + behavior without the auto-mode masking it. + capacity - goodput-weighted: each leg's share tracks its recently- + measured throughput (bytes/sec), so a fast leg carries more + and a slow one carries little — the thin-spread aggregation + mode. A just-promoted leg starts at a small cold-leg floor + share and ramps as its goodput proves out. -Affects every active and future mux'd route group on this visor. -The setting persists to skywire-config.json so it survives restart. +Affects every active and future mux'd route group on this visor +IMMEDIATELY (the router re-applies the mode to live route groups). The +setting persists to skywire-config.json so it survives restart. Example: - skywire cli proxy mux mode equal # before measuring aggregation + skywire cli proxy mux mode capacity # goodput-weighted thin spread skywire cli proxy mux info --watch 1s - skywire cli proxy mux mode auto # back to weighted`, + skywire cli proxy mux mode auto # back to latency-weighted`, Args: cobra.ExactArgs(1), DisableFlagsInUseLine: true, Run: func(cmd *cobra.Command, args []string) { mode := args[0] - if mode != "auto" && mode != "equal" { - internal.PrintFatalError(cmd.Flags(), fmt.Errorf("mode must be 'auto' or 'equal', got %q", mode)) + switch mode { + case "auto", "equal", "capacity": + default: + internal.PrintFatalError(cmd.Flags(), fmt.Errorf("mode must be 'auto', 'equal', or 'capacity', got %q", mode)) } rpcClient, err := clirpc.Client(cmd.Flags()) if err != nil { diff --git a/pkg/router/policy/preset/preset.go b/pkg/router/policy/preset/preset.go index ddf0fb9321..fe6cd0fc6a 100644 --- a/pkg/router/policy/preset/preset.go +++ b/pkg/router/policy/preset/preset.go @@ -335,9 +335,14 @@ func decideAdaptive(ctx Context, cands []Candidate) Spec { // Symmetric Mux => establishMuxRoutes builds every leg FULL-DUPLEX // (fwdCount == revCount). See the Shape note above for why bidirectional // setup replaced the old ForwardMux=1 / ReverseMux=N asymmetry. - Mux: adaptRevActive + adaptStandbyMax, + Mux: AdaptRevActive() + adaptStandbyMax, RotationIntervalSeconds: 20, - Distribution: "auto", + // Goodput-weighted thin spread: each active leg's share tracks its + // recently-measured throughput, with a cold-leg floor so a fresh leg + // ramps in (see rebuildWeights). Live-tunable to auto/equal via the + // mux-control RPC. This is the aggregation default — a slow leg carries + // little and can't stall the reorder frontier. + Distribution: "capacity", } if anyKnownTransportKind(cands) { spec.Chosen = mostTransportDiverse(cands) diff --git a/pkg/router/policy/preset/preset_test.go b/pkg/router/policy/preset/preset_test.go index aa62838e50..25cc5f8ea2 100644 --- a/pkg/router/policy/preset/preset_test.go +++ b/pkg/router/policy/preset/preset_test.go @@ -19,9 +19,9 @@ func TestDecide_ShapePresets(t *testing.T) { {"latency-adaptive", Context{App: "vpn-client"}, Spec{Mux: 5, MinHops: 2, RotationIntervalSeconds: 30, Distribution: "auto"}}, {"elastic-mux", Context{App: "skynet-client"}, Spec{Mux: 2, MinHops: 2, RotationIntervalSeconds: 20, Distribution: "auto"}}, {"probe-and-prune", Context{App: "skynet-client"}, Spec{Mux: 3, MinHops: 2, RotationIntervalSeconds: 30, Distribution: "auto"}}, - {"adaptive", Context{App: "vpn-client"}, Spec{Mux: adaptRevActive + adaptStandbyMax, RotationIntervalSeconds: 20, Distribution: "auto"}}, + {"adaptive", Context{App: "vpn-client"}, Spec{Mux: AdaptRevActive() + adaptStandbyMax, RotationIntervalSeconds: 20, Distribution: "capacity"}}, {"adaptive/chat", Context{App: "skychat"}, Spec{Mux: 1}}, - {"adaptive/custom-session", Context{App: "g8"}, Spec{Mux: adaptRevActive + adaptStandbyMax, RotationIntervalSeconds: 20, Distribution: "auto"}}, + {"adaptive/custom-session", Context{App: "g8"}, Spec{Mux: AdaptRevActive() + adaptStandbyMax, RotationIntervalSeconds: 20, Distribution: "capacity"}}, {"ledbat", Context{App: "skysocks-client"}, Spec{Mux: 3, MinHops: 2, RotationIntervalSeconds: 20, Distribution: "auto"}}, {"ledbat/chat", Context{App: "skychat"}, Spec{Mux: 1}}, } @@ -80,8 +80,8 @@ func TestDecide_Adaptive(t *testing.T) { if got.Chosen == nil || got.Chosen.Hops[0] != "b" { t.Fatalf("adaptive should seed the most transport-diverse route (b); got %+v", got.Chosen) } - if got.Mux != adaptRevActive+adaptStandbyMax || got.ForwardMux != 0 || got.ReverseMux != 0 || - got.RotationIntervalSeconds != 20 || got.Distribution != "auto" || + if got.Mux != AdaptRevActive()+adaptStandbyMax || got.ForwardMux != 0 || got.ReverseMux != 0 || + got.RotationIntervalSeconds != 20 || got.Distribution != "capacity" || got.MinHops != 0 { t.Errorf("adaptive seed shape changed: %+v", got) } @@ -119,7 +119,7 @@ func TestDecide_Adaptive(t *testing.T) { // App-agnostic: an unknown / custom-named session still gets the adaptive // bidirectional mux (not the empty spec) — the fix must apply to EVERY app that // dials a route group, not a hardcoded allowlist. - if got := Decide("adaptive", Context{App: "some-custom-app"}, nil); got.Mux != adaptRevActive+adaptStandbyMax || got.ForwardMux != 0 || got.ReverseMux != 0 { + if got := Decide("adaptive", Context{App: "some-custom-app"}, nil); got.Mux != AdaptRevActive()+adaptStandbyMax || got.ForwardMux != 0 || got.ReverseMux != 0 { t.Errorf("adaptive must apply to any non-chat app; got %+v", got) } } @@ -128,24 +128,24 @@ func TestDecide_Adaptive(t *testing.T) { // PROACTIVELY parks the surplus reverse legs as warm standby (down to the // steady active target) instead of waiting for an idle signal, and never parks // leg 0 (the primary / forward leg the router refuses to demote). The decide -// seeds adaptRevActive+adaptStandbyMax reverse legs; the router brings them up +// seeds AdaptRevActive()+adaptStandbyMax reverse legs; the router brings them up // active; the first ticks must demote the newest surplus legs to standby. func TestEngine_OnTick_AdaptiveHoldsWarmStandby(t *testing.T) { e := New() - // adaptRevActive+adaptStandbyMax legs all active (as the router first + // AdaptRevActive()+adaptStandbyMax legs all active (as the router first // establishes them — every leg is born active). Steady active target = - // adaptRevActive. - total := adaptRevActive + adaptStandbyMax + // AdaptRevActive(). + total := AdaptRevActive() + adaptStandbyMax legs := make([]LegInfo, total) for i := range legs { legs[i] = LegInfo{Index: i, TransportID: string(rune('a' + i)), Kind: "stcpr", LatencyMs: 40, Alive: true} } - // ONE tick parks the WHOLE surplus (total-adaptRevActive legs) at once — the + // ONE tick parks the WHOLE surplus (total-AdaptRevActive() legs) at once — the // bulk fast-converge that stops a wide uncapped mux from lingering as a // head-of-line-stalling active set. Leg 0 (primary) is never parked. act := e.OnTick("adaptive", legs) - wantParked := total - adaptRevActive + wantParked := total - AdaptRevActive() if len(act.DemoteToStandby) != wantParked { t.Fatalf("expected a bulk park of %d surplus legs in one tick, got %d: %+v", wantParked, len(act.DemoteToStandby), act) @@ -165,7 +165,7 @@ func TestEngine_OnTick_AdaptiveHoldsWarmStandby(t *testing.T) { t.Fatalf("expected %d distinct parked legs, got %d", wantParked, len(parked)) } - // Steady state: adaptRevActive active + adaptStandbyMax standby → no further + // Steady state: AdaptRevActive() active + adaptStandbyMax standby → no further // structural change (no dip, no churn). if act := e.OnTick("adaptive", legs); !reflect.DeepEqual(act, RotationAction{}) { t.Errorf("at steady active target the adaptive tick must be a no-op; got %+v", act) @@ -322,12 +322,18 @@ func TestEngine_OnTick_AdaptiveSwapsBadPrimary(t *testing.T) { } // TestEngine_OnTick_AdaptiveCapsActiveUnderLoad asserts the active set is capped -// at adaptCap even when the group reads SATURATED. During the uncap fill, legs +// at AdaptCap() even when the group reads SATURATED. During the uncap fill, legs // are born active and route-setup traffic keeps the group saturated — the park -// must still cap active at adaptCap (parking the slowest excess) so the reorder -// buffer never spans more than adaptCap legs; only born-active excess is shed, -// never below adaptCap under load. +// must still cap active at AdaptCap() (parking the slowest excess) so the reorder +// buffer never spans more than AdaptCap() legs; only born-active excess is shed, +// never below AdaptCap() under load. func TestEngine_OnTick_AdaptiveCapsActiveUnderLoad(t *testing.T) { + // Pin the runtime-tunable cap to 8 so this test exercises the capping logic + // deterministically regardless of the shipped default (which is now 60). + restoreCap := AdaptCap() + SetAdaptCap(8) + defer SetAdaptCap(restoreCap) + e := seedSaturatedAdaptive(1) const n = 20 legs := make([]LegInfo, n) @@ -342,10 +348,10 @@ func TestEngine_OnTick_AdaptiveCapsActiveUnderLoad(t *testing.T) { legs[i] = LegInfo{Index: i, TransportID: tid, Kind: "stcpr", LatencyMs: 20 + i*10, Alive: true, RecvBytes: recv} } act := e.OnTick("adaptive", legs) - wantParked := n - adaptCap + wantParked := n - AdaptCap() if len(act.DemoteToStandby) != wantParked { - t.Fatalf("under load, active must be capped at adaptCap=%d: want %d parked, got %d: %+v", - adaptCap, wantParked, len(act.DemoteToStandby), act) + t.Fatalf("under load, active must be capped at AdaptCap()=%d: want %d parked, got %d: %+v", + AdaptCap(), wantParked, len(act.DemoteToStandby), act) } parked := map[int]bool{} for _, idx := range act.DemoteToStandby { @@ -354,14 +360,14 @@ func TestEngine_OnTick_AdaptiveCapsActiveUnderLoad(t *testing.T) { } parked[idx] = true } - // The adaptCap kept-active legs are the fastest (leg 0 + legs 1..adaptCap-1); - // the n-adaptCap slowest (highest index) are parked. - for i := adaptCap; i < n; i++ { + // The AdaptCap() kept-active legs are the fastest (leg 0 + legs 1..AdaptCap()-1); + // the n-AdaptCap() slowest (highest index) are parked. + for i := AdaptCap(); i < n; i++ { if !parked[i] { - t.Errorf("slowest excess legs must be parked to cap active at adaptCap: leg %d (%dms) not parked", i, 20+i*10) + t.Errorf("slowest excess legs must be parked to cap active at AdaptCap(): leg %d (%dms) not parked", i, 20+i*10) } } - for i := 1; i < adaptCap; i++ { + for i := 1; i < AdaptCap(); i++ { if parked[i] { t.Errorf("fastest legs must stay active under load: leg %d (%dms) was parked", i, 20+i*10) } @@ -461,7 +467,7 @@ func TestEngine_OnTick_AdaptiveStandbyFloor(t *testing.T) { // fall below adaptStandbyMin across many heavy-load ticks, and the active // width must grow. eng := New() - active, standby := adaptRevActive, adaptStandbyMax + active, standby := AdaptRevActive(), adaptStandbyMax var recv uint64 build := func() []LegInfo { legs := make([]LegInfo, 0, active+standby) @@ -500,8 +506,8 @@ func TestEngine_OnTick_AdaptiveStandbyFloor(t *testing.T) { t.Fatalf("tick %d: standby reserve %d fell below floor %d (action %+v)", tick, standby, adaptStandbyMin, act) } } - if active <= adaptRevActive { - t.Errorf("sustained saturation should grow the active width beyond the %d-leg seed; active=%d", adaptRevActive, active) + if active <= AdaptRevActive() { + t.Errorf("sustained saturation should grow the active width beyond the %d-leg seed; active=%d", AdaptRevActive(), active) } if standby < adaptStandbyMin { t.Errorf("standby reserve ended below floor: %d < %d", standby, adaptStandbyMin) @@ -551,8 +557,19 @@ func TestEngine_OnTick_AdaptiveEvictsGrossOutlier(t *testing.T) { // width and then stops reshaping (a long tail of no-ops). This is the fix for // the observed "active set churns constantly, disrupting in-flight flows." func TestEngine_OnTick_AdaptiveStableUnderSteady(t *testing.T) { + // Pin the runtime-tunable widths to the values this test's convergence/churn + // assertions were written against (cap 8, floor 1), independent of the shipped + // defaults (now 60 / 4). Set the cap first so the low floor is not clamped, and + // restore in reverse. + restoreCap := AdaptCap() + SetAdaptCap(8) + defer SetAdaptCap(restoreCap) + restoreW := AdaptRevActive() + SetAdaptRevActive(1) + defer SetAdaptRevActive(restoreW) + eng := New() - active, standby := adaptRevActive, adaptStandbyMax + active, standby := AdaptRevActive(), adaptStandbyMax var recv uint64 build := func() []LegInfo { legs := make([]LegInfo, 0, active+standby) @@ -916,8 +933,19 @@ func (s *adaptiveSim) step() RotationAction { //nolint:unparam // test helper: r // TestEngine_OnTick_AdaptiveForwardWidensOnUpload asserts an upload-heavy flow // (growing SentBytes, flat RecvBytes) widens the FORWARD mux via AddForwardLeg // while the reverse controller stays completely lean (no AddLeg, reverse target -// unchanged at adaptRevActive). +// unchanged at AdaptRevActive()). func TestEngine_OnTick_AdaptiveForwardWidensOnUpload(t *testing.T) { + // Pin the widths to the values this test asserts against (cap 8, floor 1), + // independent of the shipped defaults (now 60 / 4). Pin BEFORE newAdaptiveSim + // so New() seeds adaptTarget from the pinned floor. Set cap first, restore in + // reverse. + restoreCap := AdaptCap() + SetAdaptCap(8) + defer SetAdaptCap(restoreCap) + restoreW := AdaptRevActive() + SetAdaptRevActive(1) + defer SetAdaptRevActive(restoreW) + s := newAdaptiveSim(1_000_000, 0) // heavy upload, zero download for i := 0; i < 20; i++ { s.step() @@ -928,8 +956,8 @@ func TestEngine_OnTick_AdaptiveForwardWidensOnUpload(t *testing.T) { if s.sawAddLeg { t.Errorf("upload-heavy flow must NOT grow the reverse/full-duplex set (AddLeg)") } - if s.e.adaptTarget != adaptRevActive { - t.Errorf("reverse target must stay lean at %d; got %d", adaptRevActive, s.e.adaptTarget) + if s.e.adaptTarget != AdaptRevActive() { + t.Errorf("reverse target must stay lean at %d; got %d", AdaptRevActive(), s.e.adaptTarget) } if s.e.adaptFwdTarget <= adaptFwdActive { t.Errorf("forward target must have widened above %d; got %d", adaptFwdActive, s.e.adaptFwdTarget) @@ -957,8 +985,8 @@ func TestEngine_OnTick_AdaptiveReverseWidensOnDownload(t *testing.T) { if s.e.adaptFwdTarget != adaptFwdActive { t.Errorf("forward target must stay lean at %d on a download flow; got %d", adaptFwdActive, s.e.adaptFwdTarget) } - if s.e.adaptTarget <= adaptRevActive { - t.Errorf("reverse target must have widened above %d; got %d", adaptRevActive, s.e.adaptTarget) + if s.e.adaptTarget <= AdaptRevActive() { + t.Errorf("reverse target must have widened above %d; got %d", AdaptRevActive(), s.e.adaptTarget) } } @@ -966,6 +994,17 @@ func TestEngine_OnTick_AdaptiveReverseWidensOnDownload(t *testing.T) { // flow collapses back to the lean single forward leg once the upload goes idle // (forward target returns to adaptFwdActive and the active set shrinks). func TestEngine_OnTick_AdaptiveForwardCollapsesOnIdle(t *testing.T) { + // Pin the widths to the values this test asserts against (cap 8, floor 1) so + // the idle steady state collapses to a SINGLE active leg, independent of the + // shipped defaults (now 60 / 4). Pin BEFORE newAdaptiveSim. Set cap first, + // restore in reverse. + restoreCap := AdaptCap() + SetAdaptCap(8) + defer SetAdaptCap(restoreCap) + restoreW := AdaptRevActive() + SetAdaptRevActive(1) + defer SetAdaptRevActive(restoreW) + s := newAdaptiveSim(1_000_000, 0) for i := 0; i < 20; i++ { // widen under upload s.step() diff --git a/pkg/router/policy/preset/tick.go b/pkg/router/policy/preset/tick.go index 64fe652f19..43e48f2a73 100644 --- a/pkg/router/policy/preset/tick.go +++ b/pkg/router/policy/preset/tick.go @@ -8,7 +8,10 @@ // original global-state behavior exactly. package preset -import "sort" +import ( + "sort" + "sync/atomic" +) // Engine holds the per-transport_id smoothing state and probe/AIMD // bookkeeping the adaptive tick controllers accumulate across @@ -108,7 +111,7 @@ func New() *Engine { adaptUnhealthy: map[string]int{}, adaptStall: map[string]int{}, adaptRecvRate: map[string]float64{}, - adaptTarget: adaptRevActive, + adaptTarget: AdaptRevActive(), adaptFwdTarget: adaptFwdActive, ledbatEWMA: map[string]float64{}, ledbatBase: map[string]float64{}, @@ -543,8 +546,6 @@ const ( // adaptRevActive + adaptStandbyMax (every leg full-duplex; forward-lean usage // is a send-side decision), and adaptTarget seeds to adaptRevActive. adaptFwdActive = 1 - adaptRevActive = 1 - adaptCap = 8 adaptAlpha = 0.3 adaptPeakDecay = 0.98 // adaptStandbyMax is the warm-standby reserve the proactive park fills to. @@ -622,7 +623,66 @@ const ( adaptThroughputOutlierFrac = 0.25 ) +// Runtime-tunable adaptive widths (see the const block above). Stored atomically +// so an operator can retune the mux's active width LIVE over the mux-control RPC +// (skywire cli proxy mux width / cap) without a rebuild — the adaptive engine +// reads them (via AdaptRevActive / AdaptCap) every tick on a per-route-group +// goroutine while the setter runs on the RPC goroutine. Defaults: a floor of 4 +// active download legs (more than one by default) and a ceiling of 60 (the +// ~50-60-leg aggregation target). +var ( + adaptRevActiveV atomic.Int64 + adaptCapV atomic.Int64 +) + +func init() { + adaptRevActiveV.Store(4) + adaptCapV.Store(60) +} + +// AdaptRevActive returns the current steady active reverse width (the floor). +func AdaptRevActive() int { return int(adaptRevActiveV.Load()) } + +// AdaptCap returns the current hard ceiling on active mux width. +func AdaptCap() int { return int(adaptCapV.Load()) } + +// SetAdaptRevActive sets the steady active reverse width (the floor the engine +// converges to when idle). Clamped to [1, AdaptCap()]. Takes effect on the next +// tick of every adaptive route group. +func SetAdaptRevActive(n int) int { + if n < 1 { + n = 1 + } + if cap := AdaptCap(); n > cap { + n = cap + } + adaptRevActiveV.Store(int64(n)) + return n +} + +// SetAdaptCap sets the hard ceiling on active mux width (the aggregation +// ceiling). Clamped to [1, adaptStandbyMax]; pulls adaptRevActive down if it +// would exceed the new cap. Takes effect on the next tick. +func SetAdaptCap(n int) int { + if n < 1 { + n = 1 + } + if n > adaptStandbyMax { + n = adaptStandbyMax + } + adaptCapV.Store(int64(n)) + if AdaptRevActive() > n { + adaptRevActiveV.Store(int64(n)) + } + return n +} + func (e *Engine) tickAdaptive(legs []LegInfo) RotationAction { + // Snapshot the runtime-tunable widths once per tick (lock-free atomic loads) + // so every read below sees a consistent value even if the mux-control RPC + // retunes them mid-tick, and so no scattered read site races the setter. + adaptCap := AdaptCap() + adaptRevActive := AdaptRevActive() for k := range e.adaptSeen { delete(e.adaptSeen, k) } diff --git a/pkg/router/route_group.go b/pkg/router/route_group.go index cb3b256f46..4f50d87f62 100644 --- a/pkg/router/route_group.go +++ b/pkg/router/route_group.go @@ -1703,6 +1703,17 @@ func (rg *RouteGroup) legDataProgressServiceFn(_ time.Duration) { len(dead), legDataProgressInterval, aggDelta, rg.mux.gapAge()) rg.pruneLivenessDeadLegs(dead) } + + // Refresh transport-selection weights on this fast cadence so + // WeightModeCapacity tracks RECENT goodput within seconds instead of the + // ~5min keep-alive cadence — essential for the weighted-ramp (a promoted leg + // earns its share within a few ticks, a fading leg loses it just as fast). + // Cheap: a per-leg byte-delta under the mux lock plus a selector rebuild. + rg.mu.Lock() + if rg.mux != nil && len(rg.tps) > 1 { + rg.mux.rebuildWeights(rg.tps) + } + rg.mu.Unlock() } // legRecvDelta is one active-or-standby leg's rg-scoped recv progress over a diff --git a/pkg/router/route_mux.go b/pkg/router/route_mux.go index f04c11bd8a..a8263deda6 100644 --- a/pkg/router/route_mux.go +++ b/pkg/router/route_mux.go @@ -55,6 +55,12 @@ const ( // observers refreshing closer than this reuse the stored rate rather than // dividing a tiny byte delta by a tiny interval into a spurious spike. goodputMinSampleNano = int64(250 * time.Millisecond) + // capacityColdFloorFrac is the floor share a just-promoted active leg gets + // under WeightModeCapacity, as a fraction of the fastest active leg's weight. + // Big enough that a fresh leg carries a measurable trickle to prove its + // goodput and ramp; small enough that a persistently slow leg stays near it + // and can't open a large reorder gap. See rebuildWeights. + capacityColdFloorFrac = 0.15 ) type legCounters struct { @@ -769,6 +775,7 @@ func (m *routeMux) rebuildWeights(tps []*transport.ManagedTransport) { if m.tpSelector.Mode() == WeightModeCapacity { m.legMu.Lock() weights := make([]float64, len(m.legs)) + var maxW float64 for i, lc := range m.legs { if lc == nil { continue @@ -776,7 +783,39 @@ func (m *routeMux) rebuildWeights(tps []*transport.ManagedTransport) { total := atomic.LoadUint64(&lc.sentBytes) + atomic.LoadUint64(&lc.recvBytes) delta := total - lc.lastTotalBytes lc.lastTotalBytes = total + // A warm-standby leg carries no send traffic — it must get zero + // weight so the scheduler never steers a packet onto a parked leg + // (which the receiver isn't expecting on that route and would stall + // the reorder frontier on). Keep sampling its byte counter above so a + // later promotion starts from a fresh delta, not a stale backlog. + if i < len(m.standby) && m.standby[i] { + weights[i] = 0 + continue + } weights[i] = float64(delta) + if weights[i] > maxW { + maxW = weights[i] + } + } + // Cold-leg floor (the weighted-RAMP): a just-promoted active leg has moved + // ~no bytes yet, so its raw delta is ~0 — under pure capacity weighting it + // would get ~no traffic and thus never accumulate the goodput it needs to + // earn a real share (a starvation deadlock). Give every active, non-standby + // leg a floor share = capacityColdFloorFrac of the fastest active leg, so a + // fresh leg carries a THIN trickle, measures its goodput, and ramps up as + // its delta grows — while a genuinely slow leg stays near the floor and can + // never open a big reorder gap. Skipped when the whole group is idle + // (maxW == 0) so an idle mux doesn't manufacture phantom weight. + if maxW > 0 { + floor := maxW * capacityColdFloorFrac + for i, lc := range m.legs { + if lc == nil || (i < len(m.standby) && m.standby[i]) { + continue + } + if weights[i] < floor { + weights[i] = floor + } + } } m.legMu.Unlock() m.tpSelector.SetCapacityWeights(weights) diff --git a/pkg/visor/api.go b/pkg/visor/api.go index 7f7a9d6aa4..32e947e4bf 100644 --- a/pkg/visor/api.go +++ b/pkg/visor/api.go @@ -139,6 +139,8 @@ type API interface { GetRouterSettings() (RouterSettings, error) SetRouterSettings(s RouterSettings) error SetMuxMode(mode string) error + SetMuxCap(n int) error + SetMuxWidth(n int) error //transports TransportTypes() ([]string, error) diff --git a/pkg/visor/api_transport.go b/pkg/visor/api_transport.go index 73482b68d3..ec240a560a 100644 --- a/pkg/visor/api_transport.go +++ b/pkg/visor/api_transport.go @@ -13,6 +13,7 @@ import ( "github.com/skycoin/skywire/pkg/app/appnet" "github.com/skycoin/skywire/pkg/cipher" "github.com/skycoin/skywire/pkg/router" + "github.com/skycoin/skywire/pkg/router/policy/preset" "github.com/skycoin/skywire/pkg/transport" types "github.com/skycoin/skywire/pkg/transport/types" ) @@ -181,6 +182,23 @@ func (v *Visor) SetMuxMode(mode string) error { return nil } +// SetMuxCap implements API. Sets the hard ceiling on adaptive mux active width +// (the aggregation ceiling) at runtime — the adaptive engine reads it on the +// next tick of every route group, so it takes effect live without a restart. +func (v *Visor) SetMuxCap(n int) error { + applied := preset.SetAdaptCap(n) + v.log.Infof("SetMuxCap: requested %d, applied %d", n, applied) + return nil +} + +// SetMuxWidth implements API. Sets the steady active download width (the floor +// the adaptive engine converges to when idle) at runtime. Takes effect live. +func (v *Visor) SetMuxWidth(n int) error { + applied := preset.SetAdaptRevActive(n) + v.log.Infof("SetMuxWidth: requested %d, applied %d", n, applied) + return nil +} + // TransportTypes implements API. func (v *Visor) TransportTypes() ([]string, error) { var tps []string diff --git a/pkg/visor/proxy_default_api.go b/pkg/visor/proxy_default_api.go index 695722809c..80dcd22d0f 100644 --- a/pkg/visor/proxy_default_api.go +++ b/pkg/visor/proxy_default_api.go @@ -398,6 +398,14 @@ func (proxyDefaultAPI) SetMuxMode(_ string) error { return ErrProxyNotSupported } +func (proxyDefaultAPI) SetMuxCap(_ int) error { + return ErrProxyNotSupported +} + +func (proxyDefaultAPI) SetMuxWidth(_ int) error { + return ErrProxyNotSupported +} + func (proxyDefaultAPI) TransportTypes() ([]string, error) { return nil, ErrProxyNotSupported } diff --git a/pkg/visor/rpc_client.go b/pkg/visor/rpc_client.go index 21706117e5..55ee21ed1f 100644 --- a/pkg/visor/rpc_client.go +++ b/pkg/visor/rpc_client.go @@ -840,6 +840,16 @@ func (rc *rpcClient) SetMuxMode(mode string) error { return rc.Call("SetMuxMode", &mode, &struct{}{}) } +// SetMuxCap sets the adaptive mux active-width ceiling at runtime. +func (rc *rpcClient) SetMuxCap(n int) error { + return rc.Call("SetMuxCap", &n, &struct{}{}) +} + +// SetMuxWidth sets the adaptive mux steady active download width at runtime. +func (rc *rpcClient) SetMuxWidth(n int) error { + return rc.Call("SetMuxWidth", &n, &struct{}{}) +} + // GetRouterSettings returns the unified runtime router knobs. func (rc *rpcClient) GetRouterSettings() (RouterSettings, error) { var out RouterSettings diff --git a/pkg/visor/rpc_client_mock.go b/pkg/visor/rpc_client_mock.go index 9ade3e580e..ebd74627c6 100644 --- a/pkg/visor/rpc_client_mock.go +++ b/pkg/visor/rpc_client_mock.go @@ -774,6 +774,14 @@ func (mc *mockRPCClient) SetMuxMode(_ string) error { return nil } +func (mc *mockRPCClient) SetMuxCap(_ int) error { + return nil +} + +func (mc *mockRPCClient) SetMuxWidth(_ int) error { + return nil +} + func (*mockRPCClient) GetRouterSettings() (RouterSettings, error) { return RouterSettings{}, nil } func (*mockRPCClient) SetRouterSettings(RouterSettings) error { return nil } diff --git a/pkg/visor/rpc_routing.go b/pkg/visor/rpc_routing.go index 192316a628..ae263b55a6 100644 --- a/pkg/visor/rpc_routing.go +++ b/pkg/visor/rpc_routing.go @@ -98,7 +98,7 @@ func (r *RPC) ServiceHealth(_ *struct{}, out *[]ServiceHealthEntry) (err error) // SetMinHops sets min_hops in visor's routing config func (r *RPC) SetMinHops(n *uint16, _ *struct{}) (err error) { - defer rpcutil.LogCall(r.log, "SetMinHops", *n) + defer rpcutil.LogCall(r.log, "SetMinHops", *n)(nil, &err) err = r.visor.SetMinHops(*n) return } @@ -149,6 +149,20 @@ func (r *RPC) SetMuxMode(mode *string, _ *struct{}) (err error) { return err } +// SetMuxCap sets the adaptive mux active-width ceiling (aggregation ceiling) +func (r *RPC) SetMuxCap(n *int, _ *struct{}) (err error) { + defer rpcutil.LogCall(r.log, "SetMuxCap", *n)(nil, &err) + err = r.visor.SetMuxCap(*n) + return err +} + +// SetMuxWidth sets the adaptive mux steady active download width (the floor) +func (r *RPC) SetMuxWidth(n *int, _ *struct{}) (err error) { + defer rpcutil.LogCall(r.log, "SetMuxWidth", *n)(nil, &err) + err = r.visor.SetMuxWidth(*n) + return err +} + // GetRouterSettings returns the unified runtime router knobs. func (r *RPC) GetRouterSettings(_ *struct{}, out *RouterSettings) (err error) { defer rpcutil.LogCall(r.log, "GetRouterSettings", nil)(out, &err)