Skip to content

perf(compaction): compact altimate-base earlier to cut prefill cost #1403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
anandgupta42 wants to merge 1 commit into from
Closed

perf(compaction): compact altimate-base earlier to cut prefill cost #1403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions packages/opencode/src/session/compaction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { Agent } from "@/agent/agent"
import { Plugin } from "@/plugin"
import { Config } from "@/config/config"
import { ProviderTransform } from "@/provider/transform"
import * as FreeTier from "@/altimate/free/client"
import { Telemetry } from "@/telemetry" // altimate_change — telemetry for compaction events
import { ModelID, ProviderID } from "@/provider/schema"
// altimate_change start — summarizer-integrity error
Expand Down Expand Up @@ -143,6 +144,10 @@ export namespace SessionCompaction {

const COMPACTION_BUFFER = 20_000

// altimate_change — 70k targets the upper tail between input p50=52k and p90=82k
// (p99=96k), aiming to reduce prefill cost above the input median.
const ALTIMATE_BASE_COMPACTION_THRESHOLD = 70_000

// altimate_change start — improved isOverflow formula with safety guard and unified headroom
// See PR #35 — fixes upstream bugs with limit.input models and small-context models
//
Expand All @@ -153,7 +158,7 @@ export namespace SessionCompaction {
// estimates are involved: estimate-derived budgets (fitHead, pin sizing) are
// computed against base * fraction, and the estimated component a caller
// passes to isOverflow is inflated by 1/fraction. PROVIDER-REPORTED usage is
// exact and is always compared against the raw limit minus headroom —
// exact and compares against the raw limit minus headroom, subject to a model cap —
// scaling exact counts by the fraction forfeited ~35% of every window for
// sessions whose counts contain no estimate at all.
// altimate_change start — was a second bare `0.65`, cross-referenced to
Expand Down Expand Up @@ -196,9 +201,23 @@ export namespace SessionCompaction {
* risks admitting more retained content than the trigger allows, which
* re-fires compaction immediately (livelock).
*/
export function overflowThreshold(input: { base: number; headroom: number; fraction: number }) {
export function overflowThreshold(input: {
base: number
headroom: number
fraction: number
model?: Pick<Provider.Model, "id" | "providerID">
}) {
const effectiveBase = effectiveContextLimit(input.base, input.fraction)
return Math.min(input.base - input.headroom, Math.max(effectiveBase - input.headroom, MIN_OVERFLOW_THRESHOLD))
const threshold = Math.min(
input.base - input.headroom,
Math.max(effectiveBase - input.headroom, MIN_OVERFLOW_THRESHOLD),
)
if (input.model?.providerID === FreeTier.PROVIDER_ID && input.model.id === FreeTier.MODEL_ID) {
// Keep estimate-derived retention budgets below the same cap, and never
// raise a stricter limit imposed by the model or configured headroom.
return Math.min(threshold, effectiveContextLimit(ALTIMATE_BASE_COMPACTION_THRESHOLD, input.fraction))
}
return threshold
}

export async function isOverflow(input: {
Expand All @@ -209,7 +228,7 @@ export namespace SessionCompaction {
* tail appended since the last provider-reported usage reading). The safety
* fraction applies ONLY to this component — it is inflated by 1/fraction to
* cover worst-case estimator undercount. `tokens` itself is provider-reported
* (exact) and is compared against the raw limit minus headroom.
* (exact) and is compared against the unscaled trigger threshold.
*/
estimatedTokens?: number
}) {
Expand All @@ -229,7 +248,7 @@ export namespace SessionCompaction {
if (base <= headroom) return false
const estimated = input.estimatedTokens ?? 0
const adjusted = estimated > 0 ? count + Math.ceil(estimated / contextSafetyFraction(config)) : count
const threshold = overflowThreshold({ base, headroom, fraction: 1 })
const threshold = overflowThreshold({ base, headroom, fraction: 1, model: input.model })
return adjusted >= threshold
}
// altimate_change end
Expand Down Expand Up @@ -302,6 +321,7 @@ export namespace SessionCompaction {
base,
headroom,
fraction: contextSafetyFraction(input.cfg),
model: input.model,
})
return Math.min(configured, Math.max(0, Math.floor(threshold * MAX_RETAINED_THRESHOLD_FRACTION)))
}
Expand Down Expand Up @@ -334,6 +354,7 @@ export namespace SessionCompaction {
base,
headroom: triggerHeadroom,
fraction: contextSafetyFraction(input.cfg),
model: input.model,
})
// altimate_change start — reserve the ledger budget only when a ledger or
// carry can actually be emitted. With both features off the reservation was
Expand Down Expand Up @@ -1174,7 +1195,7 @@ export namespace SessionCompaction {
// so an admitted pin (plus reserved buffer and working slack) can never by
// itself re-fire compaction immediately after a compaction.
if (base <= headroom) return 0
const threshold = overflowThreshold({ base, headroom, fraction: contextSafetyFraction(input.cfg) })
const threshold = overflowThreshold({ base, headroom, fraction: contextSafetyFraction(input.cfg), model: input.model })
if (threshold <= 0) return 0
const maxTokens = input.cfg.compaction?.pin_max_tokens ?? PIN_MAX_TOKENS
const fraction = input.cfg.compaction?.pin_window_fraction ?? PIN_WINDOW_FRACTION
Expand Down
145 changes: 145 additions & 0 deletions packages/opencode/test/session/compaction.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,14 @@ function createModel(opts: {
} as Provider.Model
}

function createAltimateBaseModel(): Provider.Model {
return {
...createModel({ context: 131_072, output: 65_536, npm: "@ai-sdk/openai-compatible" }),
id: ModelID.make("altimate-base"),
providerID: ProviderID.make("altimate-free"),
}
}

const testProviderConfig: Partial<ConfigV1.Info> = {
enabled_providers: ["test"],
provider: {
Expand Down Expand Up @@ -480,6 +488,143 @@ describe("session.compaction.isOverflow", () => {
})
// altimate_change end

it.live(
"altimate-base: compacts at 70000 tokens with no compaction config or input limit",
provideTmpdirInstance(() =>
Effect.gen(function* () {
const model = createAltimateBaseModel()
expect(model.limit.input).toBeUndefined()
for (const [input, expected] of [
[52_000, false],
[69_999, false],
[70_000, true],
[70_001, true],
] as const) {
const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected)
}
}),
),
)

it.live(
"altimate-base: requires both identities and preserves other models' global boundary",
provideTmpdirInstance(() =>
Effect.gen(function* () {
// Deliberately different limits: base=196608, headroom=max(20000,16384).
// A provider-only or model-only match must still compact at 176608.
const other = createModel({ context: 262_144, input: 196_608, output: 16_384 })
const models = [
other,
{ ...other, providerID: ProviderID.make("altimate-free") },
{ ...other, id: ModelID.make("altimate-base") },
]
for (const model of models) {
for (const [input, expected] of [
[70_001, false],
[176_607, false],
[176_608, true],
[176_609, true],
] as const) {
const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected)
}
}
}),
),
)

it.live(
"altimate-base: still counts total usage and the estimated tail at the lower boundary",
provideTmpdirInstance(() =>
Effect.gen(function* () {
const model = createAltimateBaseModel()
const tokens = { input: 52_000, output: 2_000, reasoning: 0, cache: { read: 15_999, write: 0 } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false)
expect(yield* compactionEffect.isOverflow({ tokens, model, estimatedTokens: 1 })).toBe(true)
expect(yield* compactionEffect.isOverflow({ tokens: { ...tokens, total: 70_000 }, model })).toBe(true)
}),
),
)

it.live(
"altimate-base: preserves stricter configured headroom",
provideTmpdirInstance(
() =>
Effect.gen(function* () {
const model = createAltimateBaseModel()
// 131072 - max(80000, 32000) = 51072, already below the new cap.
for (const [input, expected] of [
[51_071, false],
[51_072, true],
] as const) {
const tokens = { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(expected)
}
}),
{ config: { compaction: { reserved: 80_000 } } },
),
)

it.live(
"altimate-base: respects disabled auto compaction",
provideTmpdirInstance(
() =>
Effect.gen(function* () {
const model = createAltimateBaseModel()
const tokens = { input: 100_000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false)
}),
{ config: { compaction: { auto: false } } },
),
)

it.live(
"altimate-base: tolerates missing output limits and preserves disabled-window guards",
provideTmpdirInstance(() =>
Effect.gen(function* () {
const base = createAltimateBaseModel()
const tokens = { input: 70_000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }
// Runtime partial metadata still uses maxOutputTokens' existing fallback.
const partial = createAltimateBaseModel()
Reflect.deleteProperty(partial.limit, "output")
expect(yield* compactionEffect.isOverflow({ tokens, model: partial })).toBe(true)
for (const context of [0, 32_000]) {
const model = { ...base, limit: { ...base.limit, context } }
expect(yield* compactionEffect.isOverflow({ tokens, model })).toBe(false)
}
}),
),
)

test("altimate-base: shared threshold also bounds ledger, retained tail, and pin budgets", () => {
const model = createAltimateBaseModel()
const limits = { base: 131_072, headroom: 32_000, fraction: 1 }
expect(SessionCompaction.overflowThreshold(limits)).toBe(99_072)
expect(SessionCompaction.overflowThreshold({ ...limits, model })).toBe(70_000)
expect(SessionCompaction.overflowThreshold({ ...limits, model, fraction: 0.65 })).toBe(45_500)
expect(SessionCompaction.overflowThreshold({ ...limits, model, headroom: 80_000 })).toBe(51_072)

expect(
SessionCompaction.effectiveLedgerBudget({
model,
cfg: { compaction: { ledger_max_tokens: 100_000 } },
}),
).toBe(35_000)
expect(
SessionCompaction.preserveRecentBudget({
model,
cfg: { compaction: { preserve_recent_tokens: 100_000, state_ledger: false, summary_carry: false } },
}),
).toBe(35_000)
expect(
SessionCompaction.pinBudget({
model,
cfg: { compaction: { pin_max_tokens: 100_000, pin_window_fraction: 1 } },
}),
).toBe(68_000)
})

it.live(
"returns true when token count exceeds usable context",
provideTmpdirInstance(() =>
Expand Down
Loading