From 23f2a518a725ccd9b85c87ca1234024550523916 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Thu, 1 Oct 2026 18:54:42 -0700 Subject: [PATCH] perf(compaction): compact altimate-base earlier to cut prefill cost altimate-base production contexts are input p50 52k / p90 82k / p99 96k, and latency is prefill-dominated, but compaction essentially never fires (its overflow threshold is ~99k and 0% of sessions exceed it). Add an altimate-base/altimate-free-scoped overflow cap so the large-context tail compacts: - ALTIMATE_BASE_COMPACTION_THRESHOLD = 70_000 (the p75+ tail between input p50=52k and p90=82k) -- compacts the largest ~quarter of sessions, not the median, to limit the summarization/quality cost. - overflowThreshold() takes an optional model and, ONLY for FreeTier.PROVIDER_ID/MODEL_ID, caps the trigger at the 70k effective limit; all other models/providers are byte-for-byte unchanged. Threaded through every caller (isOverflow, retention-budget, pin sizing). bun test test/session/compaction.test.ts: 41 pass, 0 fail (incl. a non-altimate isolation test and just-below/above boundary tests). Co-Authored-By: Claude Opus 4.8 --- packages/opencode/src/session/compaction.ts | 33 +++- .../opencode/test/session/compaction.test.ts | 145 ++++++++++++++++++ 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index bb6a638fef..981c4f9771 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -14,6 +14,7 @@ import { Agent } from "@/agent/agent" import { Plugin } from "@/plugin" import { Config } from "@/config/config" import { ProviderTransform } from "@/provider/transform" +import * as FreeTier from "@/altimate/free/client" import { Telemetry } from "@/telemetry" // altimate_change — telemetry for compaction events import { ModelID, ProviderID } from "@/provider/schema" // altimate_change start — summarizer-integrity error @@ -143,6 +144,10 @@ export namespace SessionCompaction { const COMPACTION_BUFFER = 20_000 + // altimate_change — 70k targets the upper tail between input p50=52k and p90=82k + // (p99=96k), aiming to reduce prefill cost above the input median. + const ALTIMATE_BASE_COMPACTION_THRESHOLD = 70_000 + // altimate_change start — improved isOverflow formula with safety guard and unified headroom // See PR #35 — fixes upstream bugs with limit.input models and small-context models // @@ -153,7 +158,7 @@ export namespace SessionCompaction { // estimates are involved: estimate-derived budgets (fitHead, pin sizing) are // computed against base * fraction, and the estimated component a caller // passes to isOverflow is inflated by 1/fraction. PROVIDER-REPORTED usage is - // exact and is always compared against the raw limit minus headroom — + // exact and compares against the raw limit minus headroom, subject to a model cap — // scaling exact counts by the fraction forfeited ~35% of every window for // sessions whose counts contain no estimate at all. // altimate_change start — was a second bare `0.65`, cross-referenced to @@ -196,9 +201,23 @@ export namespace SessionCompaction { * risks admitting more retained content than the trigger allows, which * re-fires compaction immediately (livelock). */ - export function overflowThreshold(input: { base: number; headroom: number; fraction: number }) { + export function overflowThreshold(input: { + base: number + headroom: number + fraction: number + model?: Pick + }) { const effectiveBase = effectiveContextLimit(input.base, input.fraction) - return Math.min(input.base - input.headroom, Math.max(effectiveBase - input.headroom, MIN_OVERFLOW_THRESHOLD)) + const threshold = Math.min( + input.base - input.headroom, + Math.max(effectiveBase - input.headroom, MIN_OVERFLOW_THRESHOLD), + ) + if (input.model?.providerID === FreeTier.PROVIDER_ID && input.model.id === FreeTier.MODEL_ID) { + // Keep estimate-derived retention budgets below the same cap, and never + // raise a stricter limit imposed by the model or configured headroom. + return Math.min(threshold, effectiveContextLimit(ALTIMATE_BASE_COMPACTION_THRESHOLD, input.fraction)) + } + return threshold } export async function isOverflow(input: { @@ -209,7 +228,7 @@ export namespace SessionCompaction { * tail appended since the last provider-reported usage reading). The safety * fraction applies ONLY to this component — it is inflated by 1/fraction to * cover worst-case estimator undercount. `tokens` itself is provider-reported - * (exact) and is compared against the raw limit minus headroom. + * (exact) and is compared against the unscaled trigger threshold. */ estimatedTokens?: number }) { @@ -229,7 +248,7 @@ export namespace SessionCompaction { if (base <= headroom) return false const estimated = input.estimatedTokens ?? 0 const adjusted = estimated > 0 ? count + Math.ceil(estimated / contextSafetyFraction(config)) : count - const threshold = overflowThreshold({ base, headroom, fraction: 1 }) + const threshold = overflowThreshold({ base, headroom, fraction: 1, model: input.model }) return adjusted >= threshold } // altimate_change end @@ -302,6 +321,7 @@ export namespace SessionCompaction { base, headroom, fraction: contextSafetyFraction(input.cfg), + model: input.model, }) return Math.min(configured, Math.max(0, Math.floor(threshold * MAX_RETAINED_THRESHOLD_FRACTION))) } @@ -334,6 +354,7 @@ export namespace SessionCompaction { base, headroom: triggerHeadroom, fraction: contextSafetyFraction(input.cfg), + model: input.model, }) // altimate_change start — reserve the ledger budget only when a ledger or // carry can actually be emitted. With both features off the reservation was @@ -1174,7 +1195,7 @@ export namespace SessionCompaction { // so an admitted pin (plus reserved buffer and working slack) can never by // itself re-fire compaction immediately after a compaction. if (base <= headroom) return 0 - const threshold = overflowThreshold({ base, headroom, fraction: contextSafetyFraction(input.cfg) }) + const threshold = overflowThreshold({ base, headroom, fraction: contextSafetyFraction(input.cfg), model: input.model }) if (threshold <= 0) return 0 const maxTokens = input.cfg.compaction?.pin_max_tokens ?? PIN_MAX_TOKENS const fraction = input.cfg.compaction?.pin_window_fraction ?? PIN_WINDOW_FRACTION diff --git a/packages/opencode/test/session/compaction.test.ts b/packages/opencode/test/session/compaction.test.ts index 388339aa03..09d5e7955e 100644 --- a/packages/opencode/test/session/compaction.test.ts +++ b/packages/opencode/test/session/compaction.test.ts @@ -126,6 +126,14 @@ function createModel(opts: { } as Provider.Model } +function createAltimateBaseModel(): Provider.Model { + return { + ...createModel({ context: 131_072, output: 65_536, npm: "@ai-sdk/openai-compatible" }), + id: ModelID.make("altimate-base"), + providerID: ProviderID.make("altimate-free"), + } +} + const testProviderConfig: Partial = { enabled_providers: ["test"], provider: { @@ -480,6 +488,143 @@ describe("session.compaction.isOverflow", () => { }) // altimate_change end + it.live( + "altimate-base: compacts at 70000 tokens with no compaction config or input limit", + provideTmpdirInstance(() => + Effect.gen(function* () { + const model = createAltimateBaseModel() + expect(model.limit.input).toBeUndefined() + for (const [input, expected] of [ + [52_000, false], + [69_999, false], + [70_000, true], + [70_001, true], + ] as const) { + const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected) + } + }), + ), + ) + + it.live( + "altimate-base: requires both identities and preserves other models' global boundary", + provideTmpdirInstance(() => + Effect.gen(function* () { + // Deliberately different limits: base=196608, headroom=max(20000,16384). + // A provider-only or model-only match must still compact at 176608. + const other = createModel({ context: 262_144, input: 196_608, output: 16_384 }) + const models = [ + other, + { ...other, providerID: ProviderID.make("altimate-free") }, + { ...other, id: ModelID.make("altimate-base") }, + ] + for (const model of models) { + for (const [input, expected] of [ + [70_001, false], + [176_607, false], + [176_608, true], + [176_609, true], + ] as const) { + const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected) + } + } + }), + ), + ) + + it.live( + "altimate-base: still counts total usage and the estimated tail at the lower boundary", + provideTmpdirInstance(() => + Effect.gen(function* () { + const model = createAltimateBaseModel() + const tokens = { input: 52_000, output: 2_000, reasoning: 0, cache: { read: 15_999, write: 0 } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false) + expect(yield* compactionEffect.isOverflow({ tokens, model, estimatedTokens: 1 })).toBe(true) + expect(yield* compactionEffect.isOverflow({ tokens: { ...tokens, total: 70_000 }, model })).toBe(true) + }), + ), + ) + + it.live( + "altimate-base: preserves stricter configured headroom", + provideTmpdirInstance( + () => + Effect.gen(function* () { + const model = createAltimateBaseModel() + // 131072 - max(80000, 32000) = 51072, already below the new cap. + for (const [input, expected] of [ + [51_071, false], + [51_072, true], + ] as const) { + const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected) + } + }), + { config: { compaction: { reserved: 80_000 } } }, + ), + ) + + it.live( + "altimate-base: respects disabled auto compaction", + provideTmpdirInstance( + () => + Effect.gen(function* () { + const model = createAltimateBaseModel() + const tokens = { input: 100_000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false) + }), + { config: { compaction: { auto: false } } }, + ), + ) + + it.live( + "altimate-base: tolerates missing output limits and preserves disabled-window guards", + provideTmpdirInstance(() => + Effect.gen(function* () { + const base = createAltimateBaseModel() + const tokens = { input: 70_000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } + // Runtime partial metadata still uses maxOutputTokens' existing fallback. + const partial = createAltimateBaseModel() + Reflect.deleteProperty(partial.limit, "output") + expect(yield* compactionEffect.isOverflow({ tokens, model: partial })).toBe(true) + for (const context of [0, 32_000]) { + const model = { ...base, limit: { ...base.limit, context } } + expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false) + } + }), + ), + ) + + test("altimate-base: shared threshold also bounds ledger, retained tail, and pin budgets", () => { + const model = createAltimateBaseModel() + const limits = { base: 131_072, headroom: 32_000, fraction: 1 } + expect(SessionCompaction.overflowThreshold(limits)).toBe(99_072) + expect(SessionCompaction.overflowThreshold({ ...limits, model })).toBe(70_000) + expect(SessionCompaction.overflowThreshold({ ...limits, model, fraction: 0.65 })).toBe(45_500) + expect(SessionCompaction.overflowThreshold({ ...limits, model, headroom: 80_000 })).toBe(51_072) + + expect( + SessionCompaction.effectiveLedgerBudget({ + model, + cfg: { compaction: { ledger_max_tokens: 100_000 } }, + }), + ).toBe(35_000) + expect( + SessionCompaction.preserveRecentBudget({ + model, + cfg: { compaction: { preserve_recent_tokens: 100_000, state_ledger: false, summary_carry: false } }, + }), + ).toBe(35_000) + expect( + SessionCompaction.pinBudget({ + model, + cfg: { compaction: { pin_max_tokens: 100_000, pin_window_fraction: 1 } }, + }), + ).toBe(68_000) + }) + it.live( "returns true when token count exceeds usable context", provideTmpdirInstance(() =>