From e10f2a31e32176769643dd71d788f0dff79583b8 Mon Sep 17 00:00:00 2001 From: Blake Bauman Date: Wed, 11 Nov 2026 15:24:37 -0700 Subject: [PATCH 1/4] Bound timeout_minutes and stop one failed reclaim stage ending the tick. A timeout_minutes above 2^31-1 fits the u32 and overflowed the int cast in the timeout sweep's make_interval, so the query raised for as long as the run stayed running. The sweep ran before lease reclaim and orphan finalisation and they were chained with ?, so every reclaim tick on every replica ended there. - Reject timeout_minutes above 525600 (one year) at compile time, on the pipeline and on a step. - Clamp in bigint before the cast in both sweep queries, and read a snapshot value that is not a non-negative whole number as absent, so a snapshot stored before the bound cannot raise. - Run every reclaim stage regardless of the others, log each failure at error with its stage, and return the first error so the tick still reports failure. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 24 +++ crates/fiber-core/src/dag.rs | 62 +++++++- crates/fiber-core/src/store.rs | 92 +++++++---- crates/fiber-scheduler/src/lib.rs | 248 ++++++++++++++++++++++-------- docs/pipeline-yaml.md | 4 +- 5 files changed, 327 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d1da45..7313bd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ minor versions may carry breaking changes. ## [Unreleased] +A stored pipeline with a `timeout_minutes` above `525600` (one year), on the pipeline or on +a step, now fails to compile. The boot scan names each one, and a push or schedule against +it leaves a failed run saying why. Runs already started keep their snapshot and are +unaffected: the timeout sweep reads an out-of-range value in one as a year. + ### Security - **Seven RustSec advisories in the lockfile, all fixed by upgrades.** `rustls` 0.23.46 @@ -13,6 +18,25 @@ minor versions may carry breaking changes. verification could accept a distrusted intermediate), `tokio` 1.54.1 (RUSTSEC-2026-0386) and `ctutils` 0.4.8 (RUSTSEC-2026-0385). TLS is on every path that carries an agent token or a session, so the 0.6.7 agent, CLI and images are affected; upgrade all three. +- **One pipeline could stop lease reclaim for the whole instance.** `timeout_minutes` is a + `u32`, and only `0` was rejected. The timeout sweep turns it into an interval through a + Postgres `int`, so a value above 2 147 483 647 — `timeout_minutes: 3000000000` fits the + field — made that query raise `integer out of range` for as long as the run carrying it + stayed `running`. The sweep was the second of the reclaim tick's stages and they were + chained, so every tick on every replica ended there: no expired lease reclaimed, no + step or run timed out, no orphaned run finalised, in any project. `/ready` stayed `ok`, + because a failed tick is a logged error and not a dead loop. Any project `writer` could + do it with one saved pipeline and one run. Three changes, each sufficient alone: + - `timeout_minutes` above `525600` is rejected when the pipeline compiles, on the + pipeline and on a step. [pipeline-yaml](docs/pipeline-yaml.md). + - Both sweep queries clamp to that bound before the cast, so a snapshot stored by an + earlier version cannot raise. A `timeout_minutes` in a snapshot that is not a + non-negative whole number is read as absent, as the offer already reads it. + - A reclaim stage that fails no longer ends the tick. Stale agents, step timeouts, run + timeouts, expired leases and orphaned runs each run regardless of the others, and + within the last four one step or run that errors does not shield the rest. Each + failure is logged at `error` with its stage, and the tick still reports + `lease reclaim failed`, naming the stages. ### Changed diff --git a/crates/fiber-core/src/dag.rs b/crates/fiber-core/src/dag.rs index 14813b2..b82f57c 100644 --- a/crates/fiber-core/src/dag.rs +++ b/crates/fiber-core/src/dag.rs @@ -34,6 +34,17 @@ pub const DEFAULT_MAX_STEPS: usize = 500; /// bounds a cost nobody chooses on purpose. const MAX_EXPANDED_BYTES: usize = 8 * 1024 * 1024; +/// The longest `timeout_minutes` a pipeline or a step may ask for: one year. +/// +/// The field is a `u32`, and the timeout sweep turns it into a Postgres interval through +/// an `int`. A value past 2^31-1 fits the field and overflowed that cast, which failed +/// the sweep's query — for every project, on every tick, for as long as the run that +/// carried it stayed `running`. Nothing legitimately waits a year for a build, so the +/// bound costs nobody anything; the sweep clamps to it as well +/// ([`crate::Store::list_timed_out_steps`]), because a run's snapshot is immutable and +/// one stored before this check existed can still carry anything. +pub const MAX_TIMEOUT_MINUTES: u32 = 525_600; + /// The cap in force, from `FIBER_MAX_STEPS` (read once). /// /// 500 is justified on cost, not on correctness, and a generated monorepo fan-out can @@ -120,6 +131,8 @@ pub enum DagError { BadArtifactPath { step: String, path: String }, #[error("`{0}`: timeout_minutes must be at least 1")] InvalidTimeout(String), + #[error("`{0}`: timeout_minutes must be at most {MAX_TIMEOUT_MINUTES} (one year)")] + TimeoutTooLong(String), #[error( "step `{step}` image `{image}` is not a docker image reference (`name[:tag][@digest]`, optionally registry-qualified)" )] @@ -219,13 +232,21 @@ struct ExpandedCell<'a> { template: &'a StepDefinition, } +/// A `timeout_minutes` is absent or within `1..=MAX_TIMEOUT_MINUTES`. `what` names the +/// pipeline or the step in the error and is only built when there is one. +fn check_timeout(minutes: Option, what: impl FnOnce() -> String) -> Result<(), DagError> { + match minutes { + Some(0) => Err(DagError::InvalidTimeout(what())), + Some(m) if m > MAX_TIMEOUT_MINUTES => Err(DagError::TimeoutTooLong(what())), + _ => Ok(()), + } +} + pub fn compile_definition(def: &PipelineDefinition) -> Result { if def.steps.is_empty() { return Err(DagError::Empty); } - if def.timeout_minutes == Some(0) { - return Err(DagError::InvalidTimeout("pipeline".into())); - } + check_timeout(def.timeout_minutes, || "pipeline".into())?; // The repo lands on a `git remote add` command line on the agent host, before any // container exists; git's `ext::` transport would run it as a command. if let Some(ws) = &def.workspace @@ -241,9 +262,7 @@ pub fn compile_definition(def: &PipelineDefinition) -> Result, on_step: Option| { + let mut s = step("a", &[], "true"); + s.timeout_minutes = on_step; + PipelineDefinition { + name: "t".into(), + env: Default::default(), + workspace: None, + concurrency: None, + on: None, + steps: vec![s], + timeout_minutes: pipeline, + } + }; + // 3_000_000_000 fits the u32 and is past what Postgres takes as an `int`: the + // value that stalled the timeout sweep. + for too_long in [MAX_TIMEOUT_MINUTES + 1, 3_000_000_000, u32::MAX] { + let e = compile_definition(&def(Some(too_long), None)).expect_err("pipeline"); + assert!(matches!(&e, DagError::TimeoutTooLong(w) if w == "pipeline")); + let e = compile_definition(&def(None, Some(too_long))).expect_err("step"); + assert!(matches!(&e, DagError::TimeoutTooLong(w) if w == "step a")); + assert!(e.to_string().contains("at most 525600"), "{e}"); + } + // The bound itself is allowed, on both. + let dag = compile_definition(&def(Some(MAX_TIMEOUT_MINUTES), Some(MAX_TIMEOUT_MINUTES))) + .expect("a year is allowed"); + assert_eq!(dag.timeout_minutes, Some(MAX_TIMEOUT_MINUTES)); + assert_eq!(dag.steps[0].timeout_minutes, Some(MAX_TIMEOUT_MINUTES)); + } + #[test] fn compiles_levels() { let def = PipelineDefinition { diff --git a/crates/fiber-core/src/store.rs b/crates/fiber-core/src/store.rs index f61d94a..18bb82a 100644 --- a/crates/fiber-core/src/store.rs +++ b/crates/fiber-core/src/store.rs @@ -1,4 +1,6 @@ -use crate::dag::{CompiledDag, compile_definition, parse_pipeline_yaml_lenient}; +use crate::dag::{ + CompiledDag, MAX_TIMEOUT_MINUTES, compile_definition, parse_pipeline_yaml_lenient, +}; use crate::models::*; // sqlx 0.9 refuses a runtime-built query string unless it is asserted safe. Every // interpolation in this file splices a column-list `const` — PIPELINE_COLS, AGENT_COLS, @@ -1522,43 +1524,37 @@ impl Store { /// Running steps whose current attempt has exceeded its timeout plus a grace /// period (the agent is expected to fail the step itself first). The timeout comes /// from the run's definition snapshot, falling back to `default_minutes`. + /// + /// The query is total over whatever a snapshot holds, because it runs for every + /// project at once and one row that made it raise would stop the sweep for all of + /// them. A snapshot is immutable, so a run stored before the compile-time bound can + /// still carry a `timeout_minutes` past what `make_interval` takes as an `int`: + /// each operand is clamped to [`MAX_TIMEOUT_MINUTES`] while still a `bigint`, and + /// only then cast. A value that is not a non-negative whole JSON number is read as + /// absent — the default applies — which is what the offer does with it + /// (`as_u64`, in `fiber-api`'s `offer_for_step`). pub async fn list_timed_out_steps( &self, default_minutes: i64, grace_minutes: i64, ) -> Result> { - Ok(sqlx::query_as::<_, TimedOutStep>( - "SELECT s.id AS step_run_id, s.run_id, s.step_id, s.agent_id, - COALESCE(js.t, $1)::bigint AS timeout_minutes - FROM step_runs s - JOIN runs r ON r.id = s.run_id - JOIN step_attempts a ON a.step_run_id = s.id AND a.attempt = s.attempt - AND a.finished_at IS NULL - LEFT JOIN LATERAL ( - SELECT (e->>'timeout_minutes')::bigint AS t - FROM jsonb_array_elements(r.definition_snapshot->'steps') e - WHERE e->>'id' = s.step_id - LIMIT 1) js ON TRUE - WHERE s.status = 'running' - AND a.started_at < NOW() - make_interval(mins => (COALESCE(js.t, $1) + $2)::int)", - ) - .bind(default_minutes) - .bind(grace_minutes) - .fetch_all(&self.pool) - .await?) + Ok(sqlx::query_as::<_, TimedOutStep>(LIST_TIMED_OUT_STEPS_SQL) + .bind(default_minutes) + .bind(grace_minutes) + .bind(i64::from(MAX_TIMEOUT_MINUTES)) + .fetch_all(&self.pool) + .await?) } /// Running runs whose snapshot carries a whole-run `timeout_minutes` that has elapsed. + /// + /// Total in the same way as [`Self::list_timed_out_steps`]: clamped before the cast, + /// and a value that is not a non-negative whole number means the run has no limit. pub async fn list_timed_out_runs(&self) -> Result> { - Ok(sqlx::query_as::<_, (Uuid, i64)>( - "SELECT id, (definition_snapshot->>'timeout_minutes')::bigint - FROM runs - WHERE status = 'running' - AND (definition_snapshot->>'timeout_minutes') IS NOT NULL - AND started_at < NOW() - make_interval(mins => (definition_snapshot->>'timeout_minutes')::int)", - ) - .fetch_all(&self.pool) - .await?) + Ok(sqlx::query_as::<_, (Uuid, i64)>(LIST_TIMED_OUT_RUNS_SQL) + .bind(i64::from(MAX_TIMEOUT_MINUTES)) + .fetch_all(&self.pool) + .await?) } /// Reclaim running steps whose lease ran out. See [`Reclaimed`] for what comes back. @@ -2966,6 +2962,44 @@ pub fn value_to_definition(value: &Value) -> Result { Ok(serde_json::from_value(value.clone())?) } +/// `$1` default minutes, `$2` grace minutes, `$3` the clamp ([`MAX_TIMEOUT_MINUTES`]). +/// +/// The `CASE` is what keeps the `::bigint` cast off text that is not a short run of +/// digits; 18 of them always fit. `LEAST` on each operand bounds the sum at twice the +/// clamp, so neither the addition nor the `::int` can overflow whatever the snapshot or +/// the environment supplied. +const LIST_TIMED_OUT_STEPS_SQL: &str = "\ +SELECT s.id AS step_run_id, s.run_id, s.step_id, s.agent_id, + LEAST(COALESCE(js.t, $1), $3)::bigint AS timeout_minutes +FROM step_runs s +JOIN runs r ON r.id = s.run_id +JOIN step_attempts a ON a.step_run_id = s.id AND a.attempt = s.attempt + AND a.finished_at IS NULL +LEFT JOIN LATERAL ( + SELECT CASE WHEN jsonb_typeof(e->'timeout_minutes') = 'number' + AND e->>'timeout_minutes' ~ '^[0-9]{1,18}$' + THEN (e->>'timeout_minutes')::bigint END AS t + FROM jsonb_array_elements(r.definition_snapshot->'steps') e + WHERE e->>'id' = s.step_id + LIMIT 1) js ON TRUE +WHERE s.status = 'running' + AND a.started_at < NOW() - make_interval( + mins => (LEAST(COALESCE(js.t, $1), $3) + LEAST($2, $3))::int)"; + +/// `$1` the clamp ([`MAX_TIMEOUT_MINUTES`]). See [`LIST_TIMED_OUT_STEPS_SQL`]. +const LIST_TIMED_OUT_RUNS_SQL: &str = "\ +SELECT t.id, t.minutes +FROM ( + SELECT id, started_at, + CASE WHEN jsonb_typeof(definition_snapshot->'timeout_minutes') = 'number' + AND definition_snapshot->>'timeout_minutes' ~ '^[0-9]{1,18}$' + THEN LEAST((definition_snapshot->>'timeout_minutes')::bigint, $1) END + AS minutes + FROM runs + WHERE status = 'running') t +WHERE t.minutes IS NOT NULL + AND t.started_at < NOW() - make_interval(mins => t.minutes::int)"; + /// The run's definition snapshot, keyed by step id. /// /// The snapshot is a JSON array, and every lookup used to be a linear scan of it: one per diff --git a/crates/fiber-scheduler/src/lib.rs b/crates/fiber-scheduler/src/lib.rs index 7fd8d01..cbf8f54 100644 --- a/crates/fiber-scheduler/src/lib.rs +++ b/crates/fiber-scheduler/src/lib.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::{RwLock, broadcast, mpsc}; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; use uuid::Uuid; const EVENTS_CHANNEL: &str = "fiber:events"; @@ -587,49 +587,61 @@ impl Scheduler { /// Finalise runs whose every step is terminal but which are still `running`: the /// crash window between a reclaim committing a failure and propagating it. Nothing /// else revisits such a run, so the reclaim tick does. - async fn finalise_orphaned_runs(&self) -> Result<()> { - for run_id in self.store.runs_with_no_open_steps().await? { - info!(%run_id, "finalising run left running with no open steps"); - let changed = self.store.propagate_after_step(run_id).await?; - for s in &changed { - if s.status_enum() == StepStatus::Queued { - self.enqueue_step(s.id, s.labels_vec()).await?; - } - let ev = RunEvent::StepUpdated { - run_id: s.run_id, - step_run_id: s.id, - step_id: s.step_id.clone(), - status: s.status_enum(), - }; - if let Ok(payload) = serde_json::to_string(&ev) { - self.publish_event(&payload).await; + /// + /// Each run is recorded on its own: one whose propagation fails must not keep the + /// runs selected after it from being looked at, this tick and every tick after. + async fn finalise_orphaned_runs(&self, errs: &mut StageErrors) { + match self.store.runs_with_no_open_steps().await { + Ok(runs) => { + for run_id in runs { + errs.record("orphaned_runs", self.finalise_orphaned_run(run_id).await); } } - let Some(run) = self.store.get_run(run_id).await? else { - continue; - }; - if changed.is_empty() && !run.status_enum().is_terminal() { - // The query excludes unknown step statuses, so this is something new: - // say what the steps look like rather than publish a non-terminal - // RunUpdated every tick. - let statuses: Vec = self - .store - .list_step_runs(run_id) - .await? - .into_iter() - .map(|s| format!("{}={}", s.step_id, s.status)) - .collect(); - warn!(%run_id, ?statuses, "run has no open steps but did not finalise"); - continue; + Err(e) => errs.record("orphaned_runs", Err(e)), + } + } + + async fn finalise_orphaned_run(&self, run_id: Uuid) -> Result<()> { + info!(%run_id, "finalising run left running with no open steps"); + let changed = self.store.propagate_after_step(run_id).await?; + for s in &changed { + if s.status_enum() == StepStatus::Queued { + self.enqueue_step(s.id, s.labels_vec()).await?; } - let ev = RunEvent::RunUpdated { - run_id: run.id, - status: run.status_enum(), + let ev = RunEvent::StepUpdated { + run_id: s.run_id, + step_run_id: s.id, + step_id: s.step_id.clone(), + status: s.status_enum(), }; if let Ok(payload) = serde_json::to_string(&ev) { self.publish_event(&payload).await; } } + let Some(run) = self.store.get_run(run_id).await? else { + return Ok(()); + }; + if changed.is_empty() && !run.status_enum().is_terminal() { + // The query excludes unknown step statuses, so this is something new: + // say what the steps look like rather than publish a non-terminal + // RunUpdated every tick. + let statuses: Vec = self + .store + .list_step_runs(run_id) + .await? + .into_iter() + .map(|s| format!("{}={}", s.step_id, s.status)) + .collect(); + warn!(%run_id, ?statuses, "run has no open steps but did not finalise"); + return Ok(()); + } + let ev = RunEvent::RunUpdated { + run_id: run.id, + status: run.status_enum(), + }; + if let Ok(payload) = serde_json::to_string(&ev) { + self.publish_event(&payload).await; + } Ok(()) } @@ -1204,7 +1216,23 @@ impl Scheduler { } } + /// One reclaim tick: its stages, each run whatever happened to the ones before it. + /// + /// They used to be chained with `?`, so the first to fail ended the tick. One run + /// whose snapshot made the timeout query raise therefore also stopped expired + /// leases being reclaimed and stalled runs being advanced — for every project, on + /// every tick. The stages share nothing but the clock, so each failure is logged + /// where it happens and the tick still returns an error for `reclaim_loop` to report. async fn reclaim_once(&self) -> Result<()> { + let mut errs = StageErrors::default(); + errs.record("stale_agents", self.mark_stale_agents().await); + self.enforce_timeouts(&mut errs).await; + errs.record("expired_leases", self.reclaim_expired_leases().await); + self.finalise_orphaned_runs(&mut errs).await; + errs.finish() + } + + async fn mark_stale_agents(&self) -> Result<()> { let stale_secs: i64 = std::env::var("FIBER_AGENT_STALE_SECS") .ok() .and_then(|s| s.parse().ok()) @@ -1216,13 +1244,14 @@ impl Scheduler { // stopped heartbeating; drop it so the agent reconnects. Its leases are not // touched: `stale_secs` is well inside a lease, and an agent this far behind // is as likely to be waiting out a reconnect as gone. If it is gone, the - // leases expire below on a later tick. + // leases expire on a later tick. self.drop_agent_session(agent_id, "stale heartbeat — reconnect") .await; } + Ok(()) + } - self.enforce_timeouts().await?; - + async fn reclaim_expired_leases(&self) -> Result<()> { let reclaimed = self.store.requeue_expired_leases().await?; if !reclaimed.requeued.is_empty() || !reclaimed.failed.is_empty() { info!( @@ -1231,49 +1260,102 @@ impl Scheduler { "reclaimed expired leases" ); } - self.publish_reclaimed(reclaimed).await?; - - self.finalise_orphaned_runs().await + self.publish_reclaimed(reclaimed).await } } impl Scheduler { /// Server-side backstop for step and run timeouts. Agents enforce the step limit /// themselves; this catches agents that are old, hung, or gone, after a grace period. - async fn enforce_timeouts(&self) -> Result<()> { + /// + /// Step timeouts and run timeouts are separate stages, and so is each step and each + /// run within them: one that cannot be failed or cancelled must not shield the rest. + async fn enforce_timeouts(&self, errs: &mut StageErrors) { let cfg = TimeoutConfig::from_env(); - for t in self + match self .store .list_timed_out_steps(cfg.default_minutes, cfg.grace_minutes) - .await? + .await { - let Some(agent_id) = t.agent_id else { - continue; - }; - warn!(step = %t.step_id, run = %t.run_id, minutes = t.timeout_minutes, "step timed out (server backstop)"); - let error = format!("timed out after {} min", t.timeout_minutes); - // Fails the attempt (honouring retries), propagates, publishes — then asks the - // agent to kill whatever is still running for it. - self.on_step_complete( - agent_id, - t.step_run_id, - None, - StepStatus::Failed, - None, - Some(error), - ) - .await?; - self.cancel_step_on_agent(agent_id, t.step_run_id).await; + Ok(steps) => { + for t in steps { + errs.record("step_timeouts", self.fail_timed_out_step(t).await); + } + } + Err(e) => errs.record("step_timeouts", Err(e)), } - for (run_id, minutes) in self.store.list_timed_out_runs().await? { - warn!(run = %run_id, minutes, "run timed out"); - let reason = format!("run timed out after {minutes} min"); - self.cancel_run_with_reason(run_id, Some(&reason)).await?; + match self.store.list_timed_out_runs().await { + Ok(runs) => { + for (run_id, minutes) in runs { + warn!(run = %run_id, minutes, "run timed out"); + let reason = format!("run timed out after {minutes} min"); + errs.record( + "run_timeouts", + self.cancel_run_with_reason(run_id, Some(&reason)) + .await + .map(|_| ()), + ); + } + } + Err(e) => errs.record("run_timeouts", Err(e)), } + } + + async fn fail_timed_out_step(&self, t: fiber_core::TimedOutStep) -> Result<()> { + let Some(agent_id) = t.agent_id else { + return Ok(()); + }; + warn!(step = %t.step_id, run = %t.run_id, minutes = t.timeout_minutes, "step timed out (server backstop)"); + let error = format!("timed out after {} min", t.timeout_minutes); + // Fails the attempt (honouring retries), propagates, publishes — then asks the + // agent to kill whatever is still running for it. + self.on_step_complete( + agent_id, + t.step_run_id, + None, + StepStatus::Failed, + None, + Some(error), + ) + .await?; + self.cancel_step_on_agent(agent_id, t.step_run_id).await; Ok(()) } } +/// What went wrong in one reclaim tick, kept so the stages after a failure still run. +/// +/// `record` logs the failure where it happened and remembers it; `finish` hands the +/// first one back, naming every stage that failed, so the tick reports an error exactly +/// when one of its stages did. Nothing here touches the store, which is what lets the +/// "a failure does not end the tick" rule be tested without one. +#[derive(Debug, Default)] +struct StageErrors { + first: Option, + /// Stage names in the order they failed, each once. + failed: Vec<&'static str>, +} + +impl StageErrors { + fn record(&mut self, stage: &'static str, result: Result<()>) { + let Err(e) = result else { + return; + }; + error!(stage, error = %e, "reclaim stage failed; continuing with the rest of the tick"); + if !self.failed.contains(&stage) { + self.failed.push(stage); + } + self.first.get_or_insert(e); + } + + fn finish(self) -> Result<()> { + match self.first { + None => Ok(()), + Some(e) => Err(e.context(format!("reclaim stages failed: {}", self.failed.join(", ")))), + } + } +} + /// Step-timeout defaults; see docs/configuration.md. #[derive(Debug, Clone, Copy)] pub struct TimeoutConfig { @@ -1727,6 +1809,40 @@ mod tests { assert_eq!(retry_plan(StepStatus::Failed, 99, 2), None); } + #[test] + fn a_failed_reclaim_stage_does_not_hide_the_stages_after_it() { + let mut errs = StageErrors::default(); + errs.record("stale_agents", Ok(())); + errs.record( + "step_timeouts", + Err(anyhow::anyhow!("integer out of range")), + ); + // The stages after the failure are still recorded — and one of them fails too. + errs.record("expired_leases", Ok(())); + errs.record("orphaned_runs", Err(anyhow::anyhow!("pool timed out"))); + errs.record( + "orphaned_runs", + Err(anyhow::anyhow!("pool timed out again")), + ); + assert_eq!(errs.failed, ["step_timeouts", "orphaned_runs"]); + // The tick fails, with the first cause and every stage that failed named. + let e = errs.finish().expect_err("a failed stage fails the tick"); + assert_eq!(e.root_cause().to_string(), "integer out of range"); + assert_eq!( + e.to_string(), + "reclaim stages failed: step_timeouts, orphaned_runs" + ); + } + + #[test] + fn a_reclaim_tick_with_no_failed_stage_succeeds() { + let mut errs = StageErrors::default(); + for stage in ["stale_agents", "step_timeouts", "expired_leases"] { + errs.record(stage, Ok(())); + } + assert!(errs.finish().is_ok()); + } + #[test] fn the_backoff_doubles_per_attempt_and_is_capped() { assert_eq!(retry_plan(StepStatus::Failed, 1, 99), Some(2)); diff --git a/docs/pipeline-yaml.md b/docs/pipeline-yaml.md index bf195c8..4b690de 100644 --- a/docs/pipeline-yaml.md +++ b/docs/pipeline-yaml.md @@ -31,7 +31,7 @@ steps: | `workspace` | no | Git clone into the step workspace. `repo` must be an `http(s)://`, `ssh://`, `git://`, or `file://` URL, an scp-like `user@host:path`, or a path — rejected when the pipeline compiles otherwise, and again by the agent, which also pins `GIT_ALLOW_PROTOCOL`. Git's `ext::` transport runs a command, and the repo is added on the agent host before any container exists. `ref` must be one branch, tag, ref or SHA: no leading `-`, `..`, spaces, or `~ ^ : ?` | | `on` | no | Push / PR / cron / interval | | `steps` | yes | Map of step id → step (YAML) or list in JSON API | -| `timeout_minutes` | no | Whole-run wall-clock limit from run start; the run is cancelled with reason `run timed out` | +| `timeout_minutes` | no | Whole-run wall-clock limit from run start; the run is cancelled with reason `run timed out`. `1` to `525600` (one year); anything else is rejected when the pipeline compiles | | `concurrency` | no | Keep one run per group in flight; see below | In YAML, steps are usually a **map** keyed by id; the compiler fills `id` / `name` from the key when omitted. Give a map-form step its own `id:` and it is rejected — the key *is* the id, and two answers to the question is one of them going unread. @@ -106,7 +106,7 @@ API replicas still leave exactly one run in flight. | `labels` | `[]` | Matched against agent labels (all required labels must be present) | | `image` | — | Optional Docker image; agent runs in container when Docker enabled. A docker image reference only (`name[:tag][@digest]`, optionally registry-qualified): the value becomes a `docker run` argument, so anything docker could read as a flag (`-v/:/host`, `--privileged`) is rejected when the pipeline compiles and again by the agent, which also passes it after `--` | | `retries` | `0` | Extra attempts after failure (exponential backoff `2^attempt` s, max 60 s, persisted on the row). An attempt lost to an agent disconnect or lease expiry also counts, and gets one extra try: a step is failed once it has lost more than `retries + 1` leases | -| `timeout_minutes` | server default (60) | Per-attempt wall-clock limit incl. workspace prep and restores. The agent kills the step and fails the attempt (retries still apply); the server independently fails it `FIBER_STEP_TIMEOUT_GRACE_MINUTES` later if the agent did not | +| `timeout_minutes` | server default (60) | Per-attempt wall-clock limit incl. workspace prep and restores. The agent kills the step and fails the attempt (retries still apply); the server independently fails it `FIBER_STEP_TIMEOUT_GRACE_MINUTES` later if the agent did not. `1` to `525600` (one year); anything else is rejected when the pipeline compiles | | `secrets` | all | Project secrets to inject, by name. Omit for every secret (the default); `secrets: []` for none. Naming them keeps credentials out of steps that have no use for them, which matters most for a step running a third-party `image:` | | `env` | `{}` | Environment for this step; overrides the pipeline's for the same name | | `working_directory` | workspace root | Run in this subdirectory. Must stay inside the workspace — absolute paths and `..` are rejected when the pipeline compiles. **`artifacts` paths stay relative to the workspace root**, so moving a step's `working_directory` does not silently change what it publishes | From 0bfe9041d8ca68adfb11fc9d17e38ff59602238d Mon Sep 17 00:00:00 2001 From: Blake Bauman Date: Wed, 11 Nov 2026 15:26:16 -0700 Subject: [PATCH 2/4] Bound retries and make its three readers total for any stored value. retries is a u32 with no upper limit and lands in an int4 column: 4294967295 was stored as -1 (a step that never retries), 2147483647 overflowed retries + 1 inside the reclaim transaction, and 2^attempt overflowed a u64 from attempt 64 (a panic in debug, a zero backoff in release). Reject retries above 100 at compile time, saturate the column and the lost-lease decision, and use saturating_pow under the existing 60 s cap. The backoff for every attempt that used to fit is unchanged and pinned. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 21 ++++++++++-- crates/fiber-core/src/dag.rs | 50 +++++++++++++++++++++++++++++ crates/fiber-core/src/store.rs | 47 +++++++++++++++++++++++++-- crates/fiber-scheduler/src/lib.rs | 53 ++++++++++++++++++++++++++++++- docs/pipeline-yaml.md | 2 +- 5 files changed, 167 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7313bd8..1a1dd48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ minor versions may carry breaking changes. ## [Unreleased] A stored pipeline with a `timeout_minutes` above `525600` (one year), on the pipeline or on -a step, now fails to compile. The boot scan names each one, and a push or schedule against -it leaves a failed run saying why. Runs already started keep their snapshot and are +a step, or a step with `retries` above `100`, now fails to compile. The boot scan names +each one, and a push or schedule against it leaves a failed run saying why. Runs already started keep their snapshot and are unaffected: the timeout sweep reads an out-of-range value in one as a year. ### Security @@ -38,6 +38,23 @@ unaffected: the timeout sweep reads an out-of-range value in one as a year. failure is logged at `error` with its stage, and the tick still reports `lease reclaim failed`, naming the stages. +### Fixed + +- **An unbounded `retries` overflowed in three places.** The field is a `u32` with no + upper limit, and the column it lands in is an `int4`. + - `retries: 4294967295` was stored as `-1`, so the step never retried at all. + - `retries: 2147483647` made `retries + 1` overflow where an expired lease decides + whether the step gets another attempt — inside the reclaim transaction, so a debug + build panicked there on every tick. + - The backoff is `2^attempt` seconds, and from attempt 64 that does not fit: a panic in + a debug build, and in release a wrap to `0`, a retry with no backoff. + + `retries` above `100` is now rejected when the pipeline compiles, and each of the three + is total for whatever a row or a snapshot from an earlier version holds: the column + saturates, the lease decision saturates, and the backoff stays at its 60 s cap. The + backoff for every attempt that used to fit is unchanged. + [pipeline-yaml](docs/pipeline-yaml.md). + ### Changed - **The advisory gate can no longer go red unseen.** `deny` only ran on a push or pull diff --git a/crates/fiber-core/src/dag.rs b/crates/fiber-core/src/dag.rs index b82f57c..206f616 100644 --- a/crates/fiber-core/src/dag.rs +++ b/crates/fiber-core/src/dag.rs @@ -45,6 +45,17 @@ const MAX_EXPANDED_BYTES: usize = 8 * 1024 * 1024; /// one stored before this check existed can still carry anything. pub const MAX_TIMEOUT_MINUTES: u32 = 525_600; +/// The most `retries` a step may ask for. +/// +/// Each retry is another lease of an agent slot, and the backoff between them is capped +/// at a minute, so a hundred is already most of two hours of waiting on a step that +/// fails instantly. The field is a `u32` and the column an `int4`: without a bound, +/// `4294967295` was stored as `-1` (a step that never retries) and `2147483647` made +/// `retries + 1` overflow in the reclaim transaction. The readers are total on their own +/// — a run retried from an old snapshot can still carry anything — so this is what makes +/// the value mean something, not what keeps the arithmetic safe. +pub const MAX_RETRIES: u32 = 100; + /// The cap in force, from `FIBER_MAX_STEPS` (read once). /// /// 500 is justified on cost, not on correctness, and a generated monorepo fan-out can @@ -133,6 +144,8 @@ pub enum DagError { InvalidTimeout(String), #[error("`{0}`: timeout_minutes must be at most {MAX_TIMEOUT_MINUTES} (one year)")] TimeoutTooLong(String), + #[error("step `{step}` retries is {retries}; it must be at most {MAX_RETRIES}")] + TooManyRetries { step: String, retries: u32 }, #[error( "step `{step}` image `{image}` is not a docker image reference (`name[:tag][@digest]`, optionally registry-qualified)" )] @@ -263,6 +276,12 @@ pub fn compile_definition(def: &PipelineDefinition) -> Result MAX_RETRIES { + return Err(DagError::TooManyRetries { + step: step.id.clone(), + retries: step.retries, + }); + } } let mut seen = HashSet::new(); @@ -1855,6 +1874,37 @@ mod tests { assert_eq!(dag.steps[0].timeout_minutes, Some(MAX_TIMEOUT_MINUTES)); } + #[test] + fn retries_above_the_bound_are_rejected() { + let def = |retries: u32| { + let mut s = step("flaky", &[], "true"); + s.retries = retries; + PipelineDefinition { + name: "t".into(), + env: Default::default(), + workspace: None, + concurrency: None, + on: None, + steps: vec![s], + timeout_minutes: None, + } + }; + // i32::MAX overflowed `retries + 1`; u32::MAX was stored as -1. + for too_many in [MAX_RETRIES + 1, i32::MAX as u32, u32::MAX] { + let e = compile_definition(&def(too_many)).expect_err("rejected"); + assert!( + matches!(&e, DagError::TooManyRetries { step, retries } + if step == "flaky" && *retries == too_many), + "{e}" + ); + assert!(e.to_string().contains("at most 100"), "{e}"); + } + for fine in [0, 3, MAX_RETRIES] { + let dag = compile_definition(&def(fine)).expect("in range"); + assert_eq!(dag.steps[0].retries, fine); + } + } + #[test] fn compiles_levels() { let def = PipelineDefinition { diff --git a/crates/fiber-core/src/store.rs b/crates/fiber-core/src/store.rs index 18bb82a..2b8ad2f 100644 --- a/crates/fiber-core/src/store.rs +++ b/crates/fiber-core/src/store.rs @@ -2882,7 +2882,7 @@ impl StepRows { self.commands.push(step.run.clone()); self.labels.push(json!(step.labels)); self.needs.push(json!(step.needs)); - self.retries.push(step.retries as i32); + self.retries.push(retries_column(step.retries)); self.errors.push(match status { StepStatus::Skipped => Some("if: condition false".to_string()), _ => None, @@ -3325,6 +3325,16 @@ fn run_status_terminal(status: &str) -> bool { matches!(status, "succeeded" | "failed" | "cancelled" | "skipped") } +/// A compiled step's `retries` as the `step_runs.retries` column (`int4`). +/// +/// Compile bounds it at [`crate::dag::MAX_RETRIES`], but `retry_run` builds rows from a +/// stored snapshot without recompiling, and one from before the bound can hold any +/// `u32`. `as i32` wrapped the top half of that range negative — `4294967295` became +/// `-1`, a step that never retries — so an out-of-range value saturates instead. +fn retries_column(retries: u32) -> i32 { + i32::try_from(retries).unwrap_or(i32::MAX) +} + /// What a reclaim (expired lease, agent disconnect) did to the steps it found running. #[derive(Debug, Default)] pub struct Reclaimed { @@ -3349,8 +3359,12 @@ enum RequeueOutcome { /// allows `retries + 1`. The extra one is for a rolling agent restart or a network /// blip, which a `retries: 0` step must survive once — while a step that kills its /// agent every time still stops after two leases instead of being re-leased forever. +/// +/// Total for any stored `retries`: `saturating_add`, because a row written before +/// `retries` was bounded can hold `i32::MAX`, and this runs inside the reclaim +/// transaction — an overflow there panicked the tick in a debug build, every tick. fn requeue_outcome(attempt: i32, retries: i32) -> RequeueOutcome { - if attempt > retries + 1 { + if attempt > retries.saturating_add(1) { RequeueOutcome::Fail } else { RequeueOutcome::Requeue @@ -4202,6 +4216,35 @@ mod transition_tests { assert_eq!(requeue_outcome(5, 2), RequeueOutcome::Fail); } + #[test] + fn a_lost_lease_is_decided_for_any_stored_retries() { + // `retries + 1` overflowed here, inside the reclaim transaction. + assert_eq!(requeue_outcome(1, i32::MAX), RequeueOutcome::Requeue); + assert_eq!(requeue_outcome(i32::MAX, i32::MAX), RequeueOutcome::Requeue); + assert_eq!( + requeue_outcome(i32::MAX, i32::MAX - 1), + RequeueOutcome::Requeue + ); + assert_eq!( + requeue_outcome(i32::MAX, i32::MAX - 2), + RequeueOutcome::Fail + ); + // Rows written by the old wrapping cast hold -1: one lease, as before. + assert_eq!(requeue_outcome(1, -1), RequeueOutcome::Fail); + assert_eq!(requeue_outcome(1, i32::MIN), RequeueOutcome::Fail); + } + + #[test] + fn retries_reach_the_column_without_wrapping() { + for n in [0, 1, 3, 100] { + assert_eq!(retries_column(n), n as i32); + } + assert_eq!(retries_column(i32::MAX as u32), i32::MAX); + // These wrapped to i32::MIN and -1. + assert_eq!(retries_column(i32::MAX as u32 + 1), i32::MAX); + assert_eq!(retries_column(u32::MAX), i32::MAX); + } + #[test] fn a_step_without_retries_survives_one_lost_lease_and_fails_on_the_second() { // One rolling agent restart is not the step's fault; a second lost lease is the diff --git a/crates/fiber-scheduler/src/lib.rs b/crates/fiber-scheduler/src/lib.rs index cbf8f54..cfc15f1 100644 --- a/crates/fiber-scheduler/src/lib.rs +++ b/crates/fiber-scheduler/src/lib.rs @@ -1460,11 +1460,17 @@ fn slots_available(concurrency: u32, db_running: i64, reserved: u32) -> bool { /// Only a failure retries — a cancel is a person's decision and a success is done. The /// backoff doubles per attempt and is capped, and lands on the row as `not_before` so it /// survives a restart and is honoured by every instance. +/// +/// `saturating_pow`, because `attempt` is only bounded by `retries`, and a row written +/// before `retries` was bounded can carry one past 63 — where `2u64.pow` panics in a +/// debug build and wraps to 0, a retry with no backoff at all, in release. Saturated, it +/// lands on the cap like every other attempt from the sixth on. fn retry_plan(status: StepStatus, attempt: i32, retries: i32) -> Option { if status != StepStatus::Failed || attempt > retries { return None; } - Some(2u64.pow(attempt.max(1) as u32).min(MAX_RETRY_BACKOFF_SECS)) + let exponent = attempt.max(1).unsigned_abs(); + Some(2u64.saturating_pow(exponent).min(MAX_RETRY_BACKOFF_SECS)) } /// Read a positive minute count, falling back to `default` for absent, unparseable, or @@ -1856,6 +1862,51 @@ mod tests { ); } + #[test] + fn the_backoff_is_what_it_was_for_every_attempt_that_used_to_fit() { + // Pinned by value, then against the old expression across its whole safe range. + let pinned = [ + (1, 2), + (2, 4), + (3, 8), + (4, 16), + (5, 32), + (6, 60), + (7, 60), + (63, 60), + ]; + for (attempt, secs) in pinned { + assert_eq!( + retry_plan(StepStatus::Failed, attempt, i32::MAX), + Some(secs), + "attempt {attempt}" + ); + } + for attempt in 0..=63 { + let before = 2u64.pow(attempt.max(1) as u32).min(MAX_RETRY_BACKOFF_SECS); + assert_eq!( + retry_plan(StepStatus::Failed, attempt, i32::MAX), + Some(before), + "attempt {attempt}" + ); + } + } + + #[test] + fn the_backoff_is_capped_where_two_to_the_attempt_no_longer_fits() { + // 2^64 does not fit a u64: `pow` panicked here in debug and wrapped to 0 in + // release. + for attempt in [64, 65, 1_000, i32::MAX] { + assert_eq!( + retry_plan(StepStatus::Failed, attempt, i32::MAX), + Some(MAX_RETRY_BACKOFF_SECS), + "attempt {attempt}" + ); + } + // A negative attempt is not a thing, and is still an answer. + assert_eq!(retry_plan(StepStatus::Failed, i32::MIN, 3), Some(2)); + } + #[test] fn a_zeroth_attempt_still_waits_before_retrying() { // attempt is 1-based in practice; `.max(1)` guards 2^0 = 1 second, which would be diff --git a/docs/pipeline-yaml.md b/docs/pipeline-yaml.md index 4b690de..6cccfea 100644 --- a/docs/pipeline-yaml.md +++ b/docs/pipeline-yaml.md @@ -105,7 +105,7 @@ API replicas still leave exactly one run in flight. | `needs` | `[]` | Upstream step ids (DAG edges) | | `labels` | `[]` | Matched against agent labels (all required labels must be present) | | `image` | — | Optional Docker image; agent runs in container when Docker enabled. A docker image reference only (`name[:tag][@digest]`, optionally registry-qualified): the value becomes a `docker run` argument, so anything docker could read as a flag (`-v/:/host`, `--privileged`) is rejected when the pipeline compiles and again by the agent, which also passes it after `--` | -| `retries` | `0` | Extra attempts after failure (exponential backoff `2^attempt` s, max 60 s, persisted on the row). An attempt lost to an agent disconnect or lease expiry also counts, and gets one extra try: a step is failed once it has lost more than `retries + 1` leases | +| `retries` | `0` | Extra attempts after failure, `0` to `100` — more is rejected when the pipeline compiles (exponential backoff `2^attempt` s, max 60 s, persisted on the row). An attempt lost to an agent disconnect or lease expiry also counts, and gets one extra try: a step is failed once it has lost more than `retries + 1` leases | | `timeout_minutes` | server default (60) | Per-attempt wall-clock limit incl. workspace prep and restores. The agent kills the step and fails the attempt (retries still apply); the server independently fails it `FIBER_STEP_TIMEOUT_GRACE_MINUTES` later if the agent did not. `1` to `525600` (one year); anything else is rejected when the pipeline compiles | | `secrets` | all | Project secrets to inject, by name. Omit for every secret (the default); `secrets: []` for none. Naming them keeps credentials out of steps that have no use for them, which matters most for a step running a third-party `image:` | | `env` | `{}` | Environment for this step; overrides the pipeline's for the same name | From dfd8aceb52268442d7727e497092767231c4f908 Mon Sep 17 00:00:00 2001 From: Blake Bauman Date: Wed, 11 Nov 2026 15:28:37 -0700 Subject: [PATCH 3/4] Sweep runs left with pending dependents, not only all-terminal ones. A step's outcome commits in one transaction and propagate_after_step runs as the next. A crash or pool timeout between them leaves the dependents pending with nothing queued or running, and the run running forever: the agent does not resend a completion. The reclaim loop's sweep exists for this window but treated pending as open, so it only caught runs whose every step had finished. Select running runs with no queued or running step instead. Pending is never a resting state on its own (backoff, backed-out offers and unmatched labels are all queued), so propagation always moves such a run; an exhaustive test over a diamond pins that. Rename the store method and the scheduler stage to say what they do, and stop warning about a run the sweep picked just before its own propagation ran. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 15 ++- crates/fiber-core/src/store.rs | 148 +++++++++++++++++++++++++++--- crates/fiber-scheduler/src/lib.rs | 83 ++++++++++++----- docs/architecture.md | 2 +- docs/operations.md | 2 +- 5 files changed, 210 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1dd48..fabede5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,13 +33,26 @@ unaffected: the timeout sweep reads an out-of-range value in one as a year. earlier version cannot raise. A `timeout_minutes` in a snapshot that is not a non-negative whole number is read as absent, as the offer already reads it. - A reclaim stage that fails no longer ends the tick. Stale agents, step timeouts, run - timeouts, expired leases and orphaned runs each run regardless of the others, and + timeouts, expired leases and stalled runs each run regardless of the others, and within the last four one step or run that errors does not shield the rest. Each failure is logged at `error` with its stage, and the tick still reports `lease reclaim failed`, naming the stages. ### Fixed +- **A run could stay `running` forever with every remaining step `pending`.** A step's + outcome is committed in one transaction and its consequences — unlocking dependents, + finalising the run — in the next. If the process died or the pool timed out between + them, the dependents were never queued; nothing was `queued` or `running`, so no agent + would ever report on the run again, and the agent's completion is not resent. The same + gap follows `start` and `retry` when every root step is skipped or carried over. The + reclaim loop has a sweep for exactly this, but it only picked up runs with no `pending` + step either, which is the case where every step had already finished. It now picks up + every `running` run with no step `queued` or `running` and re-propagates it, on the + next tick (15 s). A `pending` step is only ever waiting on a dependency — a retry + backoff, a backed-out offer and a step waiting for a matching agent are all `queued` — + so such a run can always be moved. The sweep also no longer warns about a run it + picked in the instant before that run's own propagation ran. - **An unbounded `retries` overflowed in three places.** The field is a `u32` with no upper limit, and the column it lands in is an `int4`. - `retries: 4294967295` was stored as `-1`, so the step never retried at all. diff --git a/crates/fiber-core/src/store.rs b/crates/fiber-core/src/store.rs index 2b8ad2f..2e15b5e 100644 --- a/crates/fiber-core/src/store.rs +++ b/crates/fiber-core/src/store.rs @@ -1608,24 +1608,45 @@ impl Store { Ok(reclaimed) } - /// Running runs with no step left to run: every step is terminal, yet the run was - /// never finalised. That is the window between a reclaim committing a step as - /// `failed` and `propagate_after_step` running on it — a process that dies in - /// between leaves the run `running` with nothing that would ever revisit it, since - /// every other sweep looks for open steps. The reclaim loop feeds these back into - /// propagation. Bounded so one tick cannot stall behind a pathological backlog. + /// Running runs with nothing in flight: no step is `queued` or `running`, so no + /// agent will ever report on the run and nothing else will revisit it. Whatever is + /// left is terminal or `pending`, and only propagation moves those — the reclaim + /// loop feeds these runs back into it. /// - /// A run holding a step whose status is outside the vocabulary (a legacy row the - /// NOT VALID check tolerates) is left out: propagation reads it as `Pending`, so it - /// could never finalise and would be re-selected every tick, crowding out real - /// orphans past the limit. Such rows are the operator's to fix. - pub async fn runs_with_no_open_steps(&self) -> Result> { + /// A run gets here because a step's outcome and the propagation that follows it are + /// two transactions. A process that dies, or a pool that times out, between them + /// leaves the outcome committed and its consequences unapplied: dependents still + /// `pending`, or every step terminal and the run never finalised. The agent does not + /// resend a completion and a resend would be dropped as stale. `start_run` and + /// `retry_run` have the same gap when every root is skipped or carried over. + /// + /// `pending` is deliberately not treated as in flight (it was, and that hid the + /// first case). Nothing parks a step there to wait: a retry backoff, an offer that + /// was backed out, and a step no connected agent has the labels for are all + /// `queued` (the first two with `not_before`); `if:` is decided at the moment a step + /// would be queued; a concurrency group cancels the older run, it never holds the + /// newer one. So a `pending` step is only ever waiting on a dependency, and when no + /// step is in flight the earliest one in the DAG has every dependency terminal — + /// propagation queues or skips it. Selecting a run that another caller is about to + /// propagate is harmless: `propagate_after_step` locks the run row, guards each + /// write on the status it read, and hands each transition to exactly one caller. + /// + /// Bounded so one tick cannot stall behind a pathological backlog, oldest first. + /// That only works while every selected run leaves the set, so a run propagation + /// cannot move must not be selected. One holding a step whose status is outside + /// the vocabulary (a legacy row the NOT VALID check tolerates) is left out for that + /// reason: propagation reads it as `Pending`, it could never finalise, and it would + /// be re-selected every tick, crowding out real ones past the limit. Such rows are + /// the operator's to fix. The same goes for a `pending` step whose `needs` names a + /// step the run does not have — compile rejects that, so it takes a hand-edited + /// row — which this query does not exclude; the caller warns about it every tick. + pub async fn runs_with_nothing_in_flight(&self) -> Result> { Ok(sqlx::query_scalar::<_, Uuid>( "SELECT r.id FROM runs r WHERE r.status = 'running' AND NOT EXISTS ( SELECT 1 FROM step_runs s - WHERE s.run_id = r.id AND s.status IN ('pending', 'queued', 'running')) + WHERE s.run_id = r.id AND s.status IN ('queued', 'running')) AND NOT EXISTS ( SELECT 1 FROM step_runs s WHERE s.run_id = r.id @@ -3628,6 +3649,109 @@ mod propagate_tests { assert!(plan_transitions(&steps, &json!([])).is_empty()); } + // --- what the stalled-run sweep relies on ------------------------------------------ + // + // `runs_with_nothing_in_flight` selects running runs with no queued or running step + // and re-propagates them. That is only right if propagation always moves such a run. + + #[test] + fn a_run_whose_dependents_were_never_unlocked_plans_a_transition() { + // `a` completed and committed; the process died before propagating. Nothing is + // queued or running, so nothing but a re-propagation will ever touch the run. + let mut steps = vec![ + step("a", StepStatus::Succeeded, &[]), + step("b", StepStatus::Pending, &["a"]), + step("c", StepStatus::Pending, &["a"]), + ]; + let plan = plan_transitions(&steps, &json!([])); + assert_eq!(plan, vec![(1, Transition::Queue), (2, Transition::Queue)]); + apply(&mut steps, &plan); + // Now something is queued: the run is back in the scheduler's hands. + assert!(steps.iter().any(|s| s.status == "queued")); + } + + #[test] + fn a_pending_chain_rooted_in_a_terminal_step_resolves() { + // Every root skipped by its `if:` at start, and the propagation after the + // insert never ran: the whole chain is pending behind a terminal step. + let mut steps = vec![ + step("a", StepStatus::Skipped, &[]), + step("b", StepStatus::Pending, &["a"]), + step("c", StepStatus::Pending, &["b"]), + step("d", StepStatus::Pending, &["c"]), + ]; + assert!(run_to_fixpoint(&mut steps, &json!([])) >= 1); + assert!( + steps.iter().all(|s| s.status_enum().is_terminal()), + "{steps:?}" + ); + + // Rooted in a success instead, the head of the chain is queued and the rest + // wait on it — pending, legitimately, because a step is now in flight. + let mut steps = vec![ + step("a", StepStatus::Succeeded, &[]), + step("b", StepStatus::Pending, &["a"]), + step("c", StepStatus::Pending, &["b"]), + ]; + assert_eq!(run_to_fixpoint(&mut steps, &json!([])), 1); + assert_eq!(steps[1].status, "queued"); + assert_eq!(steps[2].status, "pending"); + } + + #[test] + fn no_run_with_nothing_in_flight_is_a_resting_state() { + // Every assignment of {pending, terminal} statuses to a diamond, queued and + // running excluded — the set the sweep selects from. Whenever a step is still + // pending, one pass must plan something, with and without an `always()` step: + // if any of these planned nothing, the sweep would re-select that run forever. + let statuses = [ + StepStatus::Pending, + StepStatus::Succeeded, + StepStatus::Failed, + StepStatus::Cancelled, + StepStatus::Skipped, + ]; + let snaps = [ + json!([]), + json!([{"id": "d", "if": "always()"}]), + json!([{"id": "a", "continue_on_error": true}, {"id": "c", "if": "always()"}]), + ]; + let mut checked = 0; + for a in statuses { + for b in statuses { + for c in statuses { + for d in statuses { + let steps = vec![ + step("a", a, &[]), + step("b", b, &["a"]), + step("c", c, &["a"]), + step("d", d, &["b", "c"]), + ]; + if !steps.iter().any(|s| s.status == "pending") { + continue; + } + for snap in &snaps { + let mut steps = steps.clone(); + assert!( + !plan_transitions(&steps, snap).is_empty(), + "nothing planned for {a:?} {b:?} {c:?} {d:?} under {snap}" + ); + // And it keeps moving until a step is in flight or all are done. + run_to_fixpoint(&mut steps, snap); + assert!( + steps.iter().any(|s| s.status == "queued") + || steps.iter().all(|s| s.status_enum().is_terminal()), + "{steps:?}" + ); + checked += 1; + } + } + } + } + } + assert_eq!(checked, (5usize.pow(4) - 4usize.pow(4)) * snaps.len()); + } + #[test] fn failure_cascades_through_a_chain_in_one_call() { // A(failed) → B → C → D: the old per-pass code could leave D pending forever. diff --git a/crates/fiber-scheduler/src/lib.rs b/crates/fiber-scheduler/src/lib.rs index cfc15f1..57c9fc7 100644 --- a/crates/fiber-scheduler/src/lib.rs +++ b/crates/fiber-scheduler/src/lib.rs @@ -584,25 +584,25 @@ impl Scheduler { Ok(()) } - /// Finalise runs whose every step is terminal but which are still `running`: the - /// crash window between a reclaim committing a failure and propagating it. Nothing - /// else revisits such a run, so the reclaim tick does. + /// Re-propagate runs that are `running` with no step queued or running: a step's + /// outcome was committed and the propagation after it never ran (a crash or a pool + /// timeout between the two transactions), so dependents were never unlocked or the + /// run never finalised. Nothing else revisits such a run, so the reclaim tick does. /// /// Each run is recorded on its own: one whose propagation fails must not keep the /// runs selected after it from being looked at, this tick and every tick after. - async fn finalise_orphaned_runs(&self, errs: &mut StageErrors) { - match self.store.runs_with_no_open_steps().await { + async fn advance_stalled_runs(&self, errs: &mut StageErrors) { + match self.store.runs_with_nothing_in_flight().await { Ok(runs) => { for run_id in runs { - errs.record("orphaned_runs", self.finalise_orphaned_run(run_id).await); + errs.record("stalled_runs", self.advance_stalled_run(run_id).await); } } - Err(e) => errs.record("orphaned_runs", Err(e)), + Err(e) => errs.record("stalled_runs", Err(e)), } } - async fn finalise_orphaned_run(&self, run_id: Uuid) -> Result<()> { - info!(%run_id, "finalising run left running with no open steps"); + async fn advance_stalled_run(&self, run_id: Uuid) -> Result<()> { let changed = self.store.propagate_after_step(run_id).await?; for s in &changed { if s.status_enum() == StepStatus::Queued { @@ -622,19 +622,30 @@ impl Scheduler { return Ok(()); }; if changed.is_empty() && !run.status_enum().is_terminal() { - // The query excludes unknown step statuses, so this is something new: - // say what the steps look like rather than publish a non-terminal - // RunUpdated every tick. - let statuses: Vec = self - .store - .list_step_runs(run_id) - .await? + let steps = self.store.list_step_runs(run_id).await?; + if !nothing_in_flight(steps.iter().map(|s| s.status_enum())) { + // Selected in the moment between a step's completion and its own + // propagation, which has since run: the run was never stalled. + debug!(%run_id, "run advanced by its own propagation before the sweep got to it"); + return Ok(()); + } + // Propagation always moves a well-formed run with nothing in flight, and + // the query excludes unknown step statuses, so this is something else — a + // `needs` naming a step the run does not have, say. Show the steps rather + // than publish a non-terminal RunUpdated every tick. + let statuses: Vec = steps .into_iter() .map(|s| format!("{}={}", s.step_id, s.status)) .collect(); - warn!(%run_id, ?statuses, "run has no open steps but did not finalise"); + warn!(%run_id, ?statuses, "run has nothing in flight and propagation cannot advance it"); return Ok(()); } + info!( + %run_id, + steps_changed = changed.len(), + status = %run.status, + "advanced a run left running with nothing in flight" + ); let ev = RunEvent::RunUpdated { run_id: run.id, status: run.status_enum(), @@ -1228,7 +1239,7 @@ impl Scheduler { errs.record("stale_agents", self.mark_stale_agents().await); self.enforce_timeouts(&mut errs).await; errs.record("expired_leases", self.reclaim_expired_leases().await); - self.finalise_orphaned_runs(&mut errs).await; + self.advance_stalled_runs(&mut errs).await; errs.finish() } @@ -1443,6 +1454,18 @@ fn completion_is_current( && attempt_is_current(reported_attempt, row_attempt) } +/// Whether none of a run's steps is queued or running — what the stalled-run sweep +/// selects on (`Store::runs_with_nothing_in_flight`), asked again after propagation +/// changed nothing. The sweep reads its list before it takes any run's lock, so a run +/// can be picked in the moment between a step completing and that completion's own +/// propagation; by the time the sweep propagates there is nothing left to do, and that +/// is a healthy run, not one to warn about every time the race is lost. +fn nothing_in_flight(steps: impl IntoIterator) -> bool { + !steps + .into_iter() + .any(|s| matches!(s, StepStatus::Queued | StepStatus::Running)) +} + /// Whether one more lease fits under `concurrency`. /// /// `db_running` is what the database shows running on the agent — the authoritative @@ -1825,21 +1848,31 @@ mod tests { ); // The stages after the failure are still recorded — and one of them fails too. errs.record("expired_leases", Ok(())); - errs.record("orphaned_runs", Err(anyhow::anyhow!("pool timed out"))); - errs.record( - "orphaned_runs", - Err(anyhow::anyhow!("pool timed out again")), - ); - assert_eq!(errs.failed, ["step_timeouts", "orphaned_runs"]); + errs.record("stalled_runs", Err(anyhow::anyhow!("pool timed out"))); + errs.record("stalled_runs", Err(anyhow::anyhow!("pool timed out again"))); + assert_eq!(errs.failed, ["step_timeouts", "stalled_runs"]); // The tick fails, with the first cause and every stage that failed named. let e = errs.finish().expect_err("a failed stage fails the tick"); assert_eq!(e.root_cause().to_string(), "integer out of range"); assert_eq!( e.to_string(), - "reclaim stages failed: step_timeouts, orphaned_runs" + "reclaim stages failed: step_timeouts, stalled_runs" ); } + #[test] + fn a_run_is_stalled_only_while_no_step_is_queued_or_running() { + use StepStatus::*; + // Pending and terminal steps are not in flight, in any mix — including none. + assert!(nothing_in_flight([])); + assert!(nothing_in_flight([Succeeded, Pending, Pending])); + assert!(nothing_in_flight([Succeeded, Failed, Cancelled, Skipped])); + // One queued or running step and the run is the scheduler's or an agent's. + assert!(!nothing_in_flight([Succeeded, Queued, Pending])); + assert!(!nothing_in_flight([Succeeded, Running, Pending])); + assert!(!nothing_in_flight([Queued])); + } + #[test] fn a_reclaim_tick_with_no_failed_stage_succeeds() { let mut errs = StageErrors::default(); diff --git a/docs/architecture.md b/docs/architecture.md index b04d349..b12333f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,7 +61,7 @@ events is survivable; dropping the socket was not, because nothing republishes a - **Offers are all or nothing.** The lease is taken first and the offer built second, from the snapshot, the project's secrets and the run's artifacts; if any of those cannot be read, nothing is sent: a store error backs the lease out (no attempt spent, a 30-second backoff so the step cannot block the queue behind it) and the pass moves on; a secret that cannot be decrypted fails the step with the reason, because that will not clear on its own. An agent is offered steps until its slots are full, oldest-queued first, and its slot count comes from `step_runs` rather than from what a replica remembers delivering. - **Concurrency groups are serialised.** A run start (or retry) in a group takes a transaction-scoped advisory lock on `(project, group)`, inserts the run with a timestamp taken *inside* the lock, and finds the runs it supersedes on the same connection; the cancels happen after the commit and are guarded like any other. Two pushes on two replicas therefore cannot each see the other as newer and both keep running. - **A lease outlives the session that took it.** An agent's WebSocket dropping — an API deploy, a proxy timeout, a blip — marks the agent offline and nothing else: its steps stay `running` under their leases, the agent keeps executing them and buffers their output, and its first heartbeat after reconnecting renews the leases. Every late message is judged by the row (`running` under this agent?), never by the session it came in on. An agent that cannot reconnect within the lease (300 s, less a margin) stops its steps without reporting them, and the reclaim loop requeues the expired leases. Two exits are deliberate and requeue at once: an agent stopping on SIGTERM says `Goodbye` after killing its steps, and a revoked token (rotate, delete, project delete) ends the session with a requeue. -- **Reclaim is bounded.** An expired lease (or a deliberate exit) requeues a step until it has lost more than `retries + 1` leases (one more than a reported failure gets, so a single rolling agent restart never fails a build); after that it fails and the run propagates, so a step that kills its agent cannot be leased forever. The reclaim loop also finalises any run left `running` with every step terminal — the window between a reclaim committing and its propagation running. +- **Reclaim is bounded.** An expired lease (or a deliberate exit) requeues a step until it has lost more than `retries + 1` leases (one more than a reported failure gets, so a single rolling agent restart never fails a build); after that it fails and the run propagates, so a step that kills its agent cannot be leased forever. The reclaim loop also re-propagates any run left `running` with no step `queued` or `running` — the window between a step's outcome committing and its propagation running, which otherwise leaves dependents `pending` (or the run unfinalised) with nothing to revisit it. Each stage of a reclaim tick runs even when an earlier one failed. - **Append-only** `step_attempts` and `log_lines` for audit. - **Durable fibers** (separate from CI DAG) persist task state in Postgres and resume after crash — see [Durable fibers](./durable-fibers.md). diff --git a/docs/operations.md b/docs/operations.md index ac39357..d8dbba1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -145,7 +145,7 @@ A run that stays `running` is one of: - **Nothing to lease it** — its steps are `queued` and no online agent matches the labels / project pool. Check **Agents** for an online agent with every required label. - **Waiting on a retry backoff** — a failed step with `retries` is `queued` with `not_before` in the future (max 60 s). - **Hung step** — the attempt exceeds its `timeout_minutes` (default `FIBER_STEP_TIMEOUT_DEFAULT_MINUTES`): the agent fails it at the deadline, and the server fails it `FIBER_STEP_TIMEOUT_GRACE_MINUTES` later if the agent did not. A whole-run `timeout_minutes` cancels the run with reason `run timed out`. -- **Agent gone** — leases expire after 5 minutes without heartbeats and the step is re-queued (`step_attempts` shows `reclaimed`). A disconnect alone does not requeue: the agent may be reconnecting, still running the step, and it renews the lease when it is back — so a step whose agent shows **offline** is not stuck until its lease has expired. A lost lease counts against the step's `retries` like a reported failure does, with one extra try: once the step has lost more than `retries + 1` leases it fails with `lease lost after N attempts` and the run finalises. A `retries: 0` step therefore survives one agent crash, while a step that reliably kills its agent (OOM, a Docker hang past the lease) stops after two leases instead of being re-leased forever. The same loop finalises any run still `running` whose every step is terminal. +- **Agent gone** — leases expire after 5 minutes without heartbeats and the step is re-queued (`step_attempts` shows `reclaimed`). A disconnect alone does not requeue: the agent may be reconnecting, still running the step, and it renews the lease when it is back — so a step whose agent shows **offline** is not stuck until its lease has expired. A lost lease counts against the step's `retries` like a reported failure does, with one extra try: once the step has lost more than `retries + 1` leases it fails with `lease lost after N attempts` and the run finalises. A `retries: 0` step therefore survives one agent crash, while a step that reliably kills its agent (OOM, a Docker hang past the lease) stops after two leases instead of being re-leased forever. The same loop picks up any run still `running` with no step `queued` or `running` — dependents left `pending`, or every step terminal, because the API stopped between recording a step's outcome and acting on it — and queues, skips or finalises it on the next tick. A failure in one part of a tick (logged at `error` as `reclaim stage failed`, with the stage) does not stop the other parts. `GET /api/steps/{id}/attempts` lists every attempt with its agent, status, and error. From 84e0874ce6a0c45ec07976369a5a4fe86671709a Mon Sep 17 00:00:00 2001 From: Blake Bauman Date: Wed, 11 Nov 2026 15:40:56 -0700 Subject: [PATCH 4/4] Read a stored timeout by one rule on both sides and clamp stored retries. Review follow-ups to the three fixes before this. - The offer and the timeout sweep disagreed on an out-of-range stored timeout_minutes: above u32::MAX the agent got the 60-minute default while the server waited a year, and a 19+ digit value was no timeout at all. Both now follow dag::stored_timeout_minutes: a whole number of at least 1, clamped to a year; anything else absent. The SQL goes through numeric, so there is no digit limit. - retries_column clamps to MAX_RETRIES instead of saturating to i32::MAX, so a retry of an old run that asked for 4294967295 does not retry for ever. - A run deleted between list_timed_out_runs and its cancel is done, not a failed stage; cancel_run_with_reason returns the typed StoreError::NotFound. - Source audits: the two timeout queries only cast a guarded, clamped value, and reclaim_once contains no ? operator. - Correct the changelog: the SQL clamp is the fix, the bound stops new values, and stage isolation limits the next such bug. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 49 +++++---- crates/fiber-api/src/ws.rs | 11 +- crates/fiber-core/src/dag.rs | 78 ++++++++++++++ crates/fiber-core/src/store.rs | 169 ++++++++++++++++++++++++------ crates/fiber-scheduler/src/lib.rs | 74 +++++++++++-- 5 files changed, 319 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fabede5..b49000e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,11 @@ minor versions may carry breaking changes. ## [Unreleased] -A stored pipeline with a `timeout_minutes` above `525600` (one year), on the pipeline or on -a step, or a step with `retries` above `100`, now fails to compile. The boot scan names -each one, and a push or schedule against it leaves a failed run saying why. Runs already started keep their snapshot and are -unaffected: the timeout sweep reads an out-of-range value in one as a year. +A stored pipeline with a `timeout_minutes` above `525600` (one year), on the pipeline or +on a step, or a step with `retries` above `100`, now fails to compile. The boot scan names +each one, and a push or schedule against it leaves a failed run saying why. Runs already +started keep their snapshot and are unaffected: a `timeout_minutes` too large in one is +enforced as a year, by the agent and by the server alike. ### Security @@ -26,17 +27,27 @@ unaffected: the timeout sweep reads an out-of-range value in one as a year. chained, so every tick on every replica ended there: no expired lease reclaimed, no step or run timed out, no orphaned run finalised, in any project. `/ready` stayed `ok`, because a failed tick is a logged error and not a dead loop. Any project `writer` could - do it with one saved pipeline and one run. Three changes, each sufficient alone: - - `timeout_minutes` above `525600` is rejected when the pipeline compiles, on the - pipeline and on a step. [pipeline-yaml](docs/pipeline-yaml.md). - - Both sweep queries clamp to that bound before the cast, so a snapshot stored by an - earlier version cannot raise. A `timeout_minutes` in a snapshot that is not a - non-negative whole number is read as absent, as the offer already reads it. - - A reclaim stage that fails no longer ends the tick. Stale agents, step timeouts, run - timeouts, expired leases and stalled runs each run regardless of the others, and - within the last four one step or run that errors does not shield the rest. Each - failure is logged at `error` with its stage, and the tick still reports - `lease reclaim failed`, naming the stages. + do it with one saved pipeline and one run. + - **The fix is in the two sweep queries.** They read the value as `numeric`, clamp it to + `525600` there, and only then cast, so no stored value can make them raise. That + covers a run that is already going, whose snapshot cannot be changed. + - **`timeout_minutes` above `525600` is rejected when the pipeline compiles,** on the + pipeline and on a step, which stops new values arriving. It does nothing for a run + already started. [pipeline-yaml](docs/pipeline-yaml.md). + - **A reclaim stage that fails no longer ends the tick,** which limits what the next + bug of this kind can do; on its own it would have left the timeout sweep failing on + every tick. Stale agents, step timeouts, run timeouts, expired leases and stalled + runs each run regardless of the others, and within the last four one step or run + that errors does not shield the rest. Each failure is logged at `error` with its + stage, and the tick still reports `lease reclaim failed`, naming the stages. A run + deleted between being found timed out and being cancelled is not a failure. + + The agent and the server now read a stored `timeout_minutes` by one rule. A whole + number of at least 1 is the timeout, clamped to a year however large; anything else — + a fraction, zero, a negative, a string — is no timeout, and the default applies (for a + whole run, no limit). Before, the offer sent an agent the 60-minute default for a + value above 4 294 967 295 while the server's backstop waited far longer for the same + step. ### Fixed @@ -63,9 +74,11 @@ unaffected: the timeout sweep reads an out-of-range value in one as a year. a debug build, and in release a wrap to `0`, a retry with no backoff. `retries` above `100` is now rejected when the pipeline compiles, and each of the three - is total for whatever a row or a snapshot from an earlier version holds: the column - saturates, the lease decision saturates, and the backoff stays at its 60 s cap. The - backoff for every attempt that used to fit is unchanged. + is total for whatever a row or a snapshot from an earlier version holds: the column is + clamped to `100`, the lease decision saturates, and the backoff stays at its 60 s cap. + Clamped, not saturated: a retry of an old run whose snapshot says `4294967295` gets + 100 retries, not two billion. The backoff for every attempt that used to fit is + unchanged. [pipeline-yaml](docs/pipeline-yaml.md). ### Changed diff --git a/crates/fiber-api/src/ws.rs b/crates/fiber-api/src/ws.rs index c59301d..544fd53 100644 --- a/crates/fiber-api/src/ws.rs +++ b/crates/fiber-api/src/ws.rs @@ -188,12 +188,11 @@ async fn offer_for_step(state: &AppState, step: StepRun) -> Result { artifacts = snapshot_str_list(s.get("artifacts")); env.extend(snapshot_env(s.get("env"))); - timeout_minutes = s - .get("timeout_minutes") - .and_then(|v| v.as_u64()) - // Out of range falls back to the default below; `as u32` wrapped 2^32 to - // 0, which the agent reads as no deadline at all. - .and_then(|m| u32::try_from(m).ok()); + // The same reading the server's timeout sweep makes of it, so the agent + // and the backstop work to one deadline: clamped to a year when too large + // (`as u32` once wrapped 2^32 to 0, which the agent reads as no deadline + // at all), and the default below when it is not a timeout. + timeout_minutes = fiber_core::dag::stored_timeout_minutes(s.get("timeout_minutes")); // Re-checked here rather than trusted from the snapshot: it was // validated when the pipeline compiled, but a snapshot is a stored // document and this is the last point before it reaches an agent. diff --git a/crates/fiber-core/src/dag.rs b/crates/fiber-core/src/dag.rs index 206f616..9f04432 100644 --- a/crates/fiber-core/src/dag.rs +++ b/crates/fiber-core/src/dag.rs @@ -245,6 +245,45 @@ struct ExpandedCell<'a> { template: &'a StepDefinition, } +/// A `timeout_minutes` as a run's stored snapshot holds it, for whoever enforces it. +/// +/// Compile only lets `1..=MAX_TIMEOUT_MINUTES` through, but a snapshot is a stored +/// document and immutable, so one written before that bound — or edited by hand — can +/// hold anything. Two places enforce the value, the offer an agent is sent and the +/// server's timeout sweep, and they have to agree or the agent and the backstop work to +/// different deadlines. The rule, here for the offer and restated in SQL for the sweep +/// (`LIST_TIMED_OUT_STEPS_SQL` / `LIST_TIMED_OUT_RUNS_SQL` in `store.rs`): +/// +/// - a whole number of at least 1 is a timeout, **clamped** to [`MAX_TIMEOUT_MINUTES`] +/// however large it is (`5000000000`, `1e20`); +/// - anything else — absent, `null`, a string, a fraction, zero, a negative — is no +/// timeout at all: `None`, and the caller's default applies. +/// +/// `5.0` is a whole number: Postgres keeps the `.0` in `jsonb` and `serde_json` reads +/// it back as a float, and the SQL side compares against `trunc`, so both take it as 5. +pub fn stored_timeout_minutes(value: Option<&serde_json::Value>) -> Option { + let serde_json::Value::Number(n) = value? else { + return None; + }; + if let Some(m) = n.as_u64() { + let clamped = u32::try_from(m) + .unwrap_or(u32::MAX) + .min(MAX_TIMEOUT_MINUTES); + return (m >= 1).then_some(clamped); + } + // Not a u64: negative, fractional, or too large for one (`jsonb` holds `1e20` as + // 21 digits, which comes back as a float). + let f = n.as_f64()?; + if !f.is_finite() || f < 1.0 || f.fract() != 0.0 { + return None; + } + Some(if f >= f64::from(MAX_TIMEOUT_MINUTES) { + MAX_TIMEOUT_MINUTES + } else { + f as u32 + }) +} + /// A `timeout_minutes` is absent or within `1..=MAX_TIMEOUT_MINUTES`. `what` names the /// pipeline or the step in the error and is only built when there is one. fn check_timeout(minutes: Option, what: impl FnOnce() -> String) -> Result<(), DagError> { @@ -1874,6 +1913,45 @@ mod tests { assert_eq!(dag.steps[0].timeout_minutes, Some(MAX_TIMEOUT_MINUTES)); } + #[test] + fn a_stored_timeout_is_clamped_or_absent_never_wrapped() { + use serde_json::json; + let read = |v: serde_json::Value| stored_timeout_minutes(Some(&v)); + // In range: itself. + assert_eq!(read(json!(1)), Some(1)); + assert_eq!(read(json!(90)), Some(90)); + assert_eq!(read(json!(MAX_TIMEOUT_MINUTES)), Some(MAX_TIMEOUT_MINUTES)); + // Whole, as Postgres hands back a `5.0` it stored. + assert_eq!(read(json!(5.0)), Some(5)); + // Too large: a year, whatever the size. 3e9 fits a u32, 5e9 a u64, 1e20 neither. + for huge in [ + json!(MAX_TIMEOUT_MINUTES + 1), + json!(3_000_000_000u64), + json!(5_000_000_000u64), + json!(u64::MAX), + json!(1e20), + serde_json::from_str("100000000000000000000").unwrap(), + ] { + assert_eq!(read(huge.clone()), Some(MAX_TIMEOUT_MINUTES), "{huge}"); + } + // Not a timeout: the default applies. + for absent in [ + json!(null), + json!("30"), + json!("abc"), + json!(1.5), + json!(0.5), + json!(0), + json!(-5), + json!(-1e20), + json!(true), + json!([30]), + ] { + assert_eq!(read(absent.clone()), None, "{absent}"); + } + assert_eq!(stored_timeout_minutes(None), None); + } + #[test] fn retries_above_the_bound_are_rejected() { let def = |retries: u32| { diff --git a/crates/fiber-core/src/store.rs b/crates/fiber-core/src/store.rs index 2e15b5e..f1a8d07 100644 --- a/crates/fiber-core/src/store.rs +++ b/crates/fiber-core/src/store.rs @@ -1,5 +1,5 @@ use crate::dag::{ - CompiledDag, MAX_TIMEOUT_MINUTES, compile_definition, parse_pipeline_yaml_lenient, + CompiledDag, MAX_RETRIES, MAX_TIMEOUT_MINUTES, compile_definition, parse_pipeline_yaml_lenient, }; use crate::models::*; // sqlx 0.9 refuses a runtime-built query string unless it is asserted safe. Every @@ -1528,11 +1528,10 @@ impl Store { /// The query is total over whatever a snapshot holds, because it runs for every /// project at once and one row that made it raise would stop the sweep for all of /// them. A snapshot is immutable, so a run stored before the compile-time bound can - /// still carry a `timeout_minutes` past what `make_interval` takes as an `int`: - /// each operand is clamped to [`MAX_TIMEOUT_MINUTES`] while still a `bigint`, and - /// only then cast. A value that is not a non-negative whole JSON number is read as - /// absent — the default applies — which is what the offer does with it - /// (`as_u64`, in `fiber-api`'s `offer_for_step`). + /// still carry a `timeout_minutes` past what `make_interval` takes as an `int`. The + /// SQL restates [`crate::dag::stored_timeout_minutes`], which the offer uses, so the + /// agent and this backstop read one value: a whole number of at least 1 is clamped + /// to [`MAX_TIMEOUT_MINUTES`], and anything else is absent and the default applies. pub async fn list_timed_out_steps( &self, default_minutes: i64, @@ -1548,8 +1547,8 @@ impl Store { /// Running runs whose snapshot carries a whole-run `timeout_minutes` that has elapsed. /// - /// Total in the same way as [`Self::list_timed_out_steps`]: clamped before the cast, - /// and a value that is not a non-negative whole number means the run has no limit. + /// Total in the same way as [`Self::list_timed_out_steps`], by the same rule: too + /// large is a year, and a value that is not a timeout means the run has no limit. pub async fn list_timed_out_runs(&self) -> Result> { Ok(sqlx::query_as::<_, (Uuid, i64)>(LIST_TIMED_OUT_RUNS_SQL) .bind(i64::from(MAX_TIMEOUT_MINUTES)) @@ -1779,7 +1778,7 @@ impl Store { .bind(run_id) .fetch_optional(&mut *tx) .await? - .ok_or_else(|| anyhow!("run not found"))?; + .ok_or_else(|| crate::StoreError::not_found("run"))?; // Locking the open steps blocks a concurrent lease until this commits; after // that the lease's `status = 'queued'` guard fails on its own. `ORDER BY id` is // the lock order every multi-row step lock in this file uses (see the reclaims), @@ -2985,10 +2984,15 @@ pub fn value_to_definition(value: &Value) -> Result { /// `$1` default minutes, `$2` grace minutes, `$3` the clamp ([`MAX_TIMEOUT_MINUTES`]). /// -/// The `CASE` is what keeps the `::bigint` cast off text that is not a short run of -/// digits; 18 of them always fit. `LEAST` on each operand bounds the sum at twice the -/// clamp, so neither the addition nor the `::int` can overflow whatever the snapshot or -/// the environment supplied. +/// The snapshot value is read by the rule in [`crate::dag::stored_timeout_minutes`]. It +/// goes through `numeric`, which holds any JSON number, and is clamped there; only the +/// clamped value is ever cast to an integer type. The outer `CASE` keeps the `::numeric` +/// cast off anything that is not a JSON number — a `CASE` evaluates only the branch it +/// takes, where the operands of an `AND` may be evaluated in either order. `LEAST` on +/// each operand of the sum bounds it at twice the clamp, so neither the addition nor +/// the `::int` can overflow whatever the snapshot or the environment supplied. +/// +/// `store::timeout_sql_tests` pins these properties against the text. const LIST_TIMED_OUT_STEPS_SQL: &str = "\ SELECT s.id AS step_run_id, s.run_id, s.step_id, s.agent_id, LEAST(COALESCE(js.t, $1), $3)::bigint AS timeout_minutes @@ -2997,9 +3001,12 @@ JOIN runs r ON r.id = s.run_id JOIN step_attempts a ON a.step_run_id = s.id AND a.attempt = s.attempt AND a.finished_at IS NULL LEFT JOIN LATERAL ( - SELECT CASE WHEN jsonb_typeof(e->'timeout_minutes') = 'number' - AND e->>'timeout_minutes' ~ '^[0-9]{1,18}$' - THEN (e->>'timeout_minutes')::bigint END AS t + SELECT CASE WHEN jsonb_typeof(e->'timeout_minutes') = 'number' THEN + CASE WHEN (e->>'timeout_minutes')::numeric >= 1 + AND (e->>'timeout_minutes')::numeric + = trunc((e->>'timeout_minutes')::numeric) + THEN LEAST((e->>'timeout_minutes')::numeric, $3)::bigint END + END AS t FROM jsonb_array_elements(r.definition_snapshot->'steps') e WHERE e->>'id' = s.step_id LIMIT 1) js ON TRUE @@ -3012,14 +3019,17 @@ const LIST_TIMED_OUT_RUNS_SQL: &str = "\ SELECT t.id, t.minutes FROM ( SELECT id, started_at, - CASE WHEN jsonb_typeof(definition_snapshot->'timeout_minutes') = 'number' - AND definition_snapshot->>'timeout_minutes' ~ '^[0-9]{1,18}$' - THEN LEAST((definition_snapshot->>'timeout_minutes')::bigint, $1) END - AS minutes + CASE WHEN jsonb_typeof(definition_snapshot->'timeout_minutes') = 'number' THEN + CASE WHEN (definition_snapshot->>'timeout_minutes')::numeric >= 1 + AND (definition_snapshot->>'timeout_minutes')::numeric + = trunc((definition_snapshot->>'timeout_minutes')::numeric) + THEN LEAST((definition_snapshot->>'timeout_minutes')::numeric, $1)::bigint + END + END AS minutes FROM runs WHERE status = 'running') t WHERE t.minutes IS NOT NULL - AND t.started_at < NOW() - make_interval(mins => t.minutes::int)"; + AND t.started_at < NOW() - make_interval(mins => LEAST(t.minutes, $1)::int)"; /// The run's definition snapshot, keyed by step id. /// @@ -3348,12 +3358,13 @@ fn run_status_terminal(status: &str) -> bool { /// A compiled step's `retries` as the `step_runs.retries` column (`int4`). /// -/// Compile bounds it at [`crate::dag::MAX_RETRIES`], but `retry_run` builds rows from a -/// stored snapshot without recompiling, and one from before the bound can hold any -/// `u32`. `as i32` wrapped the top half of that range negative — `4294967295` became -/// `-1`, a step that never retries — so an out-of-range value saturates instead. +/// Compile bounds it at [`MAX_RETRIES`], but `retry_run` builds rows from a stored +/// snapshot without recompiling, and one from before the bound can hold any `u32`. +/// `as i32` wrapped the top half of that range negative — `4294967295` became `-1`, a +/// step that never retries. Clamped to the bound rather than saturated to `i32::MAX`: +/// re-running such a run must not turn "never retries" into "retries for ever". fn retries_column(retries: u32) -> i32 { - i32::try_from(retries).unwrap_or(i32::MAX) + i32::try_from(retries.min(MAX_RETRIES)).unwrap_or(i32::MAX) } /// What a reclaim (expired lease, agent disconnect) did to the steps it found running. @@ -3590,6 +3601,97 @@ fn plan_transitions_indexed( plan } +#[cfg(test)] +mod timeout_sql_tests { + //! The timeout sweep's two queries run for every project on every reclaim tick, so + //! one that can raise on a stored value stops the sweep for the whole instance. + //! Postgres is not available to a unit test; what can be pinned is the text — the + //! properties that make the queries total, so an edit that drops one goes red here. + use super::*; + + const BOTH: [(&str, &str); 2] = [ + ("steps", LIST_TIMED_OUT_STEPS_SQL), + ("runs", LIST_TIMED_OUT_RUNS_SQL), + ]; + + /// Every place the raw snapshot value is cast, as `(cast type, text before it)`. + fn raw_casts(sql: &str) -> Vec<&str> { + sql.match_indices("'timeout_minutes')::") + .map(|(i, m)| { + let rest = &sql[i + m.len()..]; + let end = rest + .find(|c: char| !c.is_ascii_alphanumeric()) + .unwrap_or(rest.len()); + &rest[..end] + }) + .collect() + } + + #[test] + fn the_raw_snapshot_value_is_only_ever_cast_to_numeric() { + // `::int` overflowed past 2^31-1 and `::bigint` raises on `1.5`, `1e20` and + // text; `numeric` holds any JSON number. + for (name, sql) in BOTH { + let casts = raw_casts(sql); + assert!(!casts.is_empty(), "{name}: the snapshot value is not read"); + assert!( + casts.iter().all(|c| *c == "numeric"), + "{name}: raw snapshot value cast to {casts:?}" + ); + } + } + + #[test] + fn the_numeric_cast_is_guarded_by_the_json_type() { + // The guard is its own CASE, opened before the first cast: an `AND` beside the + // cast would not promise the order. + for (name, sql) in BOTH { + let guard = sql + .find("jsonb_typeof(") + .unwrap_or_else(|| panic!("{name}: no jsonb_typeof guard")); + let then = guard + + sql[guard..] + .find("= 'number' THEN") + .expect("guard is a CASE arm"); + let first_cast = sql.find("'timeout_minutes')::").expect("a cast"); + assert!(then < first_cast, "{name}: a cast precedes the type guard"); + } + } + + #[test] + fn every_integer_cast_is_of_a_clamped_value() { + for (name, sql) in BOTH { + // The value leaves `numeric` only through LEAST(.., clamp). + // So every `::bigint` closes a parenthesis on a line that opened a LEAST — + // `x::numeric::bigint`, the unclamped form, closes nothing. + let to_bigint: Vec<_> = sql.match_indices("::bigint").collect(); + assert!(!to_bigint.is_empty(), "{name}"); + for (i, _) in to_bigint { + let line_start = sql[..i].rfind('\n').map_or(0, |n| n + 1); + assert!( + sql[line_start..i].ends_with(')') && sql[line_start..i].contains("LEAST("), + "{name}: `::bigint` of something not clamped: {}", + &sql[line_start..i] + ); + } + // And reaches `make_interval`'s int exactly once, clamped there too. + let to_int: Vec<_> = sql + .match_indices("::int") + .filter(|(i, _)| !sql[i + 5..].starts_with(|c: char| c.is_ascii_alphanumeric())) + .collect(); + assert_eq!(to_int.len(), 1, "{name}: {to_int:?}"); + let (i, _) = to_int[0]; + let interval = sql.rfind("make_interval(").expect("make_interval"); + assert!(interval < i, "{name}: `::int` outside make_interval"); + assert!( + sql[interval..i].contains("LEAST("), + "{name}: make_interval takes an unclamped value: {}", + &sql[interval..i] + ); + } + } +} + #[cfg(test)] mod propagate_tests { use super::*; @@ -4359,14 +4461,17 @@ mod transition_tests { } #[test] - fn retries_reach_the_column_without_wrapping() { - for n in [0, 1, 3, 100] { + fn retries_reach_the_column_clamped_never_wrapped() { + for n in [0, 1, 3, MAX_RETRIES] { assert_eq!(retries_column(n), n as i32); } - assert_eq!(retries_column(i32::MAX as u32), i32::MAX); - // These wrapped to i32::MIN and -1. - assert_eq!(retries_column(i32::MAX as u32 + 1), i32::MAX); - assert_eq!(retries_column(u32::MAX), i32::MAX); + // Only a snapshot from before the bound holds these. They wrapped to i32::MIN + // and -1; a retry of that run gets the most a pipeline may now ask for. + let most = MAX_RETRIES as i32; + assert_eq!(retries_column(MAX_RETRIES + 1), most); + assert_eq!(retries_column(i32::MAX as u32), most); + assert_eq!(retries_column(i32::MAX as u32 + 1), most); + assert_eq!(retries_column(u32::MAX), most); } #[test] diff --git a/crates/fiber-scheduler/src/lib.rs b/crates/fiber-scheduler/src/lib.rs index 57c9fc7..ba87234 100644 --- a/crates/fiber-scheduler/src/lib.rs +++ b/crates/fiber-scheduler/src/lib.rs @@ -1300,12 +1300,8 @@ impl Scheduler { for (run_id, minutes) in runs { warn!(run = %run_id, minutes, "run timed out"); let reason = format!("run timed out after {minutes} min"); - errs.record( - "run_timeouts", - self.cancel_run_with_reason(run_id, Some(&reason)) - .await - .map(|_| ()), - ); + let cancelled = self.cancel_run_with_reason(run_id, Some(&reason)).await; + errs.record("run_timeouts", gone_is_done(cancelled.map(|_| ()))); } } Err(e) => errs.record("run_timeouts", Err(e)), @@ -1334,6 +1330,25 @@ impl Scheduler { } } +/// A sweep's action on a row it listed a moment ago: `NotFound` means retention or a +/// project delete removed the row in between, and there is nothing left to act on. +/// That is the sweep's work done for it, not a stage that failed. Judged on the typed +/// [`fiber_core::StoreError`], never on the message. +fn gone_is_done(result: Result<()>) -> Result<()> { + match result { + Err(e) + if matches!( + e.downcast_ref::(), + Some(fiber_core::StoreError::NotFound(_)) + ) => + { + debug!(error = %e, "swept row no longer exists"); + Ok(()) + } + other => other, + } +} + /// What went wrong in one reclaim tick, kept so the stages after a failure still run. /// /// `record` logs the failure where it happened and remembers it; `finish` hands the @@ -1873,6 +1888,53 @@ mod tests { assert!(!nothing_in_flight([Queued])); } + #[test] + fn no_stage_of_a_reclaim_tick_can_end_it_early() { + // `StageErrors` only helps if every stage goes through it. A `?` in + // `reclaim_once` is how one failed stage used to end the tick for the rest; + // nothing but the source can say there is none, so read it. + let src = include_str!("lib.rs"); + let start = src + .find(" async fn reclaim_once(&self) -> Result<()> {") + .expect("reclaim_once"); + let body = &src[start..]; + let body = &body[..body.find("\n }\n").expect("end of reclaim_once")]; + assert!( + body.contains("errs.finish()"), + "not the function expected:\n{body}" + ); + assert!( + body.matches(".await").count() >= 4, + "expected the tick's stages here:\n{body}" + ); + assert!( + !body.contains('?'), + "a `?` in reclaim_once lets one stage end the tick:\n{body}" + ); + assert!( + !body.contains("return"), + "an early return in reclaim_once:\n{body}" + ); + } + + #[test] + fn a_swept_row_that_was_deleted_meanwhile_is_not_a_failed_stage() { + let gone: Result<()> = Err(fiber_core::StoreError::not_found("run").into()); + assert!(gone_is_done(gone).is_ok()); + // With context added on the way up, it is still the same typed error. + let gone: Result<()> = Err( + anyhow::Error::from(fiber_core::StoreError::not_found("run")) + .context("cancelling a timed-out run"), + ); + assert!(gone_is_done(gone).is_ok()); + // Anything else is a failure — including one that merely says "not found". + let lookalike: Result<()> = Err(anyhow::anyhow!("run not found")); + assert!(gone_is_done(lookalike).is_err()); + let other: Result<()> = Err(fiber_core::StoreError::Validation("no".into()).into()); + assert!(gone_is_done(other).is_err()); + assert!(gone_is_done(Ok(())).is_ok()); + } + #[test] fn a_reclaim_tick_with_no_failed_stage_succeeds() { let mut errs = StageErrors::default();