diff --git a/crates/data/src/query.rs b/crates/data/src/query.rs index 47ea19005d..a8ee8d8d19 100644 --- a/crates/data/src/query.rs +++ b/crates/data/src/query.rs @@ -19,7 +19,8 @@ //! list matches nothing, so `2 NOT IN (1, NULL)` is true where SQL makes it unknown. A `LIKE` //! too costly to match ([`MAX_LIKE_STEPS`]) is unknown, as is any that would take an expression's //! `LIKE`s past what they may take for one row together ([`MAX_ROW_LIKE_STEPS`]). Arithmetic on a -//! null, a division by zero or an overflow gives null; `+` joins text that isn't a number. +//! null, a division by zero or an overflow gives null ([`value_or_fault`] says which error it +//! was); `+` joins text that isn't a number. //! //! The vocabulary lives in catalogs the parser reads, with a signature and description for //! each entry: [`functions`], [`VARS`] and [`Kw`]. Whatever helps write expressions reads them too, @@ -131,6 +132,57 @@ pub enum Var { Measure(Measure, f64), } +/// Why an expression's value is null although nothing it read was: the errors that give null +/// rather than stop the expression ([`value_or_fault`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Fault { + /// `x / 0`, `x % 0`, `0 ^ -1`. + DivisionByZero, + /// `sqrt` of a number below zero. + NegativeRoot, + /// `ln` or `log10` of zero or a number below zero. + Logarithm, + /// A result past the largest number a value holds (`exp(1000)`, `10 ^ 400`). + Overflow, + /// A result that isn't a number (`(-8) ^ 0.5`), or a stored one that isn't (NaN, infinity). + NotANumber, + /// Text that isn't a number where one is needed: `-`, `*`, `/`, `%` or `^`, or a number + /// argument (`sqrt(name)`, `to_real('abc')`). + Text, + /// A date where a number is needed: arithmetic, or a number argument. + Date, + /// `clamp(low, x, high)` with `low` above `high`. + ClampBounds, +} + +impl Fault { + /// What happened, to follow "because of" in a notice. + pub fn describe(self) -> &'static str { + match self { + Fault::DivisionByZero => "a division by zero", + Fault::NegativeRoot => "the square root of a negative number", + Fault::Logarithm => "the logarithm of zero or a negative number", + Fault::Overflow => "a result too large to represent", + Fault::NotANumber => "a value that isn't a finite number", + Fault::Text => "text that isn't a number", + Fault::Date => "a date used as a number", + Fault::ClampBounds => "a clamp whose low bound is above its high bound", + } + } + + /// The fault of using `v` as a number, when it isn't missing and doesn't read as one. Empty + /// or `` text is missing, as a write reads it ([`Value::asks_null`]), not an error. + pub fn of_non_number(v: &Value) -> Option { + match v { + _ if v.asks_null() => None, + _ if v.as_f64().is_some() => None, + Value::Text(_) => Some(Fault::Text), + Value::Date(_) | Value::DateTime(_) => Some(Fault::Date), + _ => Some(Fault::NotANumber), + } + } +} + /// Values an expression reads from where it runs. #[derive(Clone, Copy, Debug, Default, PartialEq)] pub struct Env { @@ -1491,6 +1543,17 @@ pub fn value(e: &Expr, ds: &Dataset, row: usize, env: &Env) -> Value { Eval::new(ds, row, env).value(e).into_owned() } +/// [`value`], with the error that made it null: `None` for any other value, and for a null the +/// expression wrote, read from a field or took from an unmet condition. An error inside an `if` +/// or `CASE` condition only picks the branch, so it isn't the result's. Where several errors +/// reach the result, the first one met is given. +pub fn value_or_fault(e: &Expr, ds: &Dataset, row: usize, env: &Env) -> (Value, Option) { + let eval = Eval::new(ds, row, env); + let v = eval.value(e).into_owned(); + let fault = if v.is_null() { eval.fault.get() } else { None }; + (v, fault) +} + /// Whether a value counts as true: a non-zero number, non-empty text; null is unknown. fn truth(v: &Value) -> Option { match v { @@ -1526,36 +1589,69 @@ fn whole(f: f64) -> Value { if f.is_finite() && f.abs() < 9e15 { Value::Int(f as i64) } else { real(f) } } -/// `a op b`: null with a null or a date, exact on whole numbers where it can be, `+` joining text. -fn arith(op: Arith, a: &Value, b: &Value) -> Value { +/// Whether `v` is empty or `` text, which a number read takes as missing +/// ([`Value::asks_null`]); a null is told apart by whether it carries an error. +fn blank_text(v: &Value) -> bool { + matches!(v, Value::Text(_)) && v.asks_null() +} + +/// A float result, or why it is null: too large, or not a number. +fn checked(f: f64) -> Result { + if f.is_finite() { + Ok(Value::Float(f)) + } else if f.is_nan() { + Err(Fault::NotANumber) + } else { + Err(Fault::Overflow) + } +} + +/// `a op b`: null with a null, exact on whole numbers where it can be, `+` joining text. An +/// error (a date, text that isn't a number, a division by zero, an overflow) gives null too, +/// and says which it was. +fn arith(op: Arith, a: &Value, b: &Value) -> Result { + if a.is_null() || b.is_null() { + return Ok(Value::Null); + } // Arithmetic on dates, which the parser refuses on a date field, is null from anything else // that gives a date (`coalesce(issued, closed) + 1`). - if a.is_null() || b.is_null() || a.temporal_key().is_some() || b.temporal_key().is_some() { - return Value::Null; + if a.temporal_key().is_some() || b.temporal_key().is_some() { + return Err(Fault::Date); } let (Some(x), Some(y)) = (a.as_f64(), b.as_f64()) else { - return if op == Arith::Add { text(format!("{a}{b}")) } else { Value::Null }; + // `+` joins text, but a stored NaN or infinity isn't text to join (`inf + 1` isn't `inf1`). + if op == Arith::Add { + if a.is_non_finite() || b.is_non_finite() { + return Err(Fault::NotANumber); + } + return Ok(text(format!("{a}{b}"))); + } + // Neither is at fault when the one that isn't a number is missing (empty text). + return Fault::of_non_number(a).or(Fault::of_non_number(b)).map_or(Ok(Value::Null), Err); }; if let (Value::Int(i), Value::Int(j)) = (a, b) { let exact = match op { Arith::Add => i.checked_add(*j), Arith::Sub => i.checked_sub(*j), Arith::Mul => i.checked_mul(*j), - Arith::Mod => return i.checked_rem(*j).map(Value::Int).unwrap_or(Value::Null), + Arith::Mod if *j == 0 => return Err(Fault::DivisionByZero), + // Exact with a divisor that isn't 0: `i64::MIN % -1` is 0. + Arith::Mod => return Ok(Value::Int(i.wrapping_rem(*j))), _ => None, }; if let Some(v) = exact { - return Value::Int(v); + return Ok(Value::Int(v)); } } match op { - Arith::Add => real(x + y), - Arith::Sub => real(x - y), - Arith::Mul => real(x * y), - Arith::Div if y != 0.0 => real(x / y), - Arith::Mod if y != 0.0 => real(x % y), - Arith::Pow => real(x.powf(y)), - _ => Value::Null, + Arith::Add => checked(x + y), + Arith::Sub => checked(x - y), + Arith::Mul => checked(x * y), + Arith::Div | Arith::Mod if y == 0.0 => Err(Fault::DivisionByZero), + Arith::Div => checked(x / y), + Arith::Mod => checked(x % y), + Arith::Pow if x == 0.0 && y < 0.0 => Err(Fault::DivisionByZero), + Arith::Pow => checked(x.powf(y)), } } @@ -1588,16 +1684,74 @@ struct Eval<'a> { env: &'a Env, /// The steps left for the expression's `LIKE`s on this row ([`MAX_ROW_LIKE_STEPS`] in all). like_steps: Cell, + /// The first error that gave null on this row ([`value_or_fault`]). + fault: Cell>, } impl<'a> Eval<'a> { fn new(ds: &'a Dataset, row: usize, env: &'a Env) -> Self { - Eval { ds, row, env, like_steps: Cell::new(MAX_ROW_LIKE_STEPS) } + Eval { ds, row, env, like_steps: Cell::new(MAX_ROW_LIKE_STEPS), fault: Cell::new(None) } + } + + /// Null, for the error `fault`, kept when it is the row's first. + fn fail(&self, fault: Fault) -> Value { + if self.fault.get().is_none() { + self.fault.set(Some(fault)); + } + Value::Null + } + + /// A result that may have failed ([`checked`], [`arith`]): its value, or null and the fault kept. + fn or_fail(&self, v: Result) -> Value { + v.unwrap_or_else(|f| self.fail(f)) + } + + /// An operator's operand `e`: its value, and whether it is missing (null without an error of + /// its own). Its error, if any, joins the errors met so far. + fn operand(&self, e: &'a Expr) -> (Cow<'a, Value>, bool) { + let before = self.fault.take(); + let v = self.value(e); + let own = self.fault.get(); + self.fault.set(before.or(own)); + let missing = v.is_null() && own.is_none(); + (v, missing) + } + + /// `v` as a number argument: text or a date that isn't one is an error, a null just null. + fn num(&self, v: &Value) -> Option { + if let Some(fault) = Fault::of_non_number(v) { + self.fail(fault); + } + v.as_f64() + } + + /// `f`, whose errors only decide something (a branch, an arm, what `nullif` compares with) + /// and so aren't kept as the result's. + fn deciding(&self, f: impl FnOnce() -> T) -> T { + let kept = self.fault.get(); + let out = f(); + self.fault.set(kept); + out + } + + /// Whether the condition `e` holds, for choosing a branch ([`deciding`](Self::deciding)). + fn branch(&self, e: &'a Expr) -> Option { + self.deciding(|| self.cond(e)) } /// SQL's three-valued logic: `None` is "unknown", what a comparison with a null gives. `NOT` - /// leaves it unknown, so `NOT (pop > 100)` doesn't match a row whose `pop` is null. + /// leaves it unknown, so `NOT (pop > 100)` doesn't match a row whose `pop` is null. An error + /// inside `e` stays the row's only while `e` is unknown, as [`value`](Self::value) keeps one. fn cond(&self, e: &'a Expr) -> Option { + let outer = self.fault.take(); + let hit = self.cond_of(e); + let inner = self.fault.get(); + self.fault.set(if hit.is_none() { outer.or(inner) } else { outer }); + hit + } + + /// [`cond`](Self::cond), before keeping or dropping the errors met inside it. + fn cond_of(&self, e: &'a Expr) -> Option { let (ds, row) = (self.ds, self.row); match e { Expr::And(terms) => { @@ -1679,7 +1833,18 @@ impl<'a> Eval<'a> { } /// `e`'s value on this row: null where it can't be worked out (a missing field, an error). + /// An error inside `e` stays the row's only while `e` is null: `coalesce(a / 0, 0) / b`, with + /// `b` null, is null because of `b`, not the division `coalesce` replaced. fn value(&self, e: &'a Expr) -> Cow<'a, Value> { + let outer = self.fault.take(); + let v = self.value_of(e); + let inner = self.fault.get(); + self.fault.set(if v.is_null() { outer.or(inner) } else { outer }); + v + } + + /// [`value`](Self::value), before keeping or dropping the errors met inside it. + fn value_of(&self, e: &'a Expr) -> Cow<'a, Value> { match e { Expr::Lit(v) => Cow::Borrowed(v), Expr::Field(i) => Cow::Borrowed(self.ds.value(self.row, *i)), @@ -1698,32 +1863,48 @@ impl<'a> Eval<'a> { .map(|v| real(v / size)) .unwrap_or(Value::Null), }), + // An operand that is missing (null without an error) makes the result null whatever + // else failed, so an error is the result's only when no operand is missing: + // `1 / 0 + pop`, with `pop` null, is null because of `pop`. + // Empty or `` text is missing to every operator but `+`, which joins it. Expr::Chain(first, rest) => { - let mut acc = self.value(first).into_owned(); + let (first, mut missing) = self.operand(first); + let mut acc = first.into_owned(); for (op, t) in rest { - acc = arith(*op, &acc, &self.value(t)); + let (v, m) = self.operand(t); + // The result so far counts too: `'' + '' - x` takes `-` on empty text. + missing |= m || (*op != Arith::Add && (blank_text(&acc) || blank_text(&v))); + acc = self.or_fail(arith(*op, &acc, &v)); + } + if missing { + self.fault.set(None); } Cow::Owned(acc) } Expr::Concat(terms) => { let mut s = String::new(); + let mut null = false; for t in terms { - let v = self.value(t); - if v.is_null() { + let (v, missing) = self.operand(t); + if missing { + self.fault.set(None); return Cow::Owned(Value::Null); } - if s.len() <= MAX_TEXT { + // Null from an error: the rest are read only to see whether one is missing. + null |= v.is_null(); + if !null && s.len() <= MAX_TEXT { s.push_str(&v.to_string()); } } - Cow::Owned(text(s)) + Cow::Owned(if null { Value::Null } else { text(s) }) } Expr::Case { subject, arms, otherwise } => { - let subject = subject.as_ref().map(|s| self.value(s)); + // The subject only picks the arm. + let subject = subject.as_ref().map(|s| self.deciding(|| self.value(s))); for (when, then) in arms { let hit = match &subject { - Some(s) => compare(s, Op::Eq, &self.value(when)), - None => self.cond(when), + Some(s) => self.deciding(|| compare(s, Op::Eq, &self.value(when))), + None => self.branch(when), }; if hit == Some(true) { return self.value(then); @@ -1740,23 +1921,59 @@ impl<'a> Eval<'a> { } /// The value of the function `f` over `args` on this row. + /// As for an operator, a missing argument (null without an error of its own) is why a null + /// result is null, whatever else failed: `clamp(pop, 1 / 0, 5)`, with `pop` null. Not for + /// the functions that skip nulls (`coalesce`, `min`, `max`): there each null argument is a + /// cause, and fixing the error alone gives a value (`coalesce(1 / 0, pop)`). fn call(&self, f: Func, args: &'a [Expr]) -> Cow<'a, Value> { - let arg = |i: usize| args.get(i).map(|a| self.value(a)).unwrap_or(Cow::Owned(Value::Null)); - let n = |i: usize| arg(i).as_f64(); + let missing = Cell::new(false); + let v = self.call_with(f, args, &missing); + if missing.get() && v.is_null() && !matches!(f, Func::Coalesce | Func::Min | Func::Max) { + self.fault.set(None); + } + v + } + + /// [`call`](Self::call), noting in `missing` an argument read that was missing. + fn call_with(&self, f: Func, args: &'a [Expr], missing: &Cell) -> Cow<'a, Value> { + let read = |a: &'a Expr| { + let (v, m) = self.operand(a); + if m { + missing.set(true); + } + v + }; + let arg = |i: usize| args.get(i).map(read).unwrap_or(Cow::Owned(Value::Null)); + // A number argument; empty or `` text there is missing, as a null is. + let number = |v: &Value| { + if blank_text(v) { + missing.set(true); + } + self.num(v) + }; + let n = |i: usize| number(&arg(i)); let txt = |i: usize| { let v = arg(i); (!v.is_null()).then(|| v.to_string()) }; - let num1 = |g: fn(f64) -> f64| Cow::Owned(n(0).map(|x| real(g(x))).unwrap_or(Value::Null)); + let num1 = |g: fn(f64) -> f64| Cow::Owned(n(0).map(|x| self.or_fail(checked(g(x)))).unwrap_or(Value::Null)); + // A function of positive numbers only (`sqrt`, `ln`): `fault` below its domain. + let num_from = |g: fn(f64) -> f64, zero_ok: bool, fault: Fault| { + Cow::Owned(match n(0) { + Some(x) if x < 0.0 || (x == 0.0 && !zero_ok) => self.fail(fault), + Some(x) => self.or_fail(checked(g(x))), + None => Value::Null, + }) + }; let text1 = |g: fn(&str) -> String| Cow::Owned(txt(0).map(|s| text(g(&s))).unwrap_or(Value::Null)); Cow::Owned(match f { Func::If => { - let pick = if args.first().and_then(|c| self.cond(c)) == Some(true) { 1 } else { 2 }; + let pick = if args.first().and_then(|c| self.branch(c)) == Some(true) { 1 } else { 2 }; return arg(pick); } Func::Coalesce => { for a in args { - let v = self.value(a); + let v = read(a); if !v.is_null() { return v; } @@ -1764,7 +1981,9 @@ impl<'a> Eval<'a> { Value::Null } Func::NullIf => { - let (a, b) = (arg(0), arg(1)); + // `b` is only compared with. + let b = || args.get(1).map(|b| self.value(b)).unwrap_or(Cow::Owned(Value::Null)); + let (a, b) = (arg(0), self.deciding(b)); if compare(&a, Op::Eq, &b) == Some(true) { Value::Null } else { @@ -1772,14 +1991,18 @@ impl<'a> Eval<'a> { } } Func::Abs => match &*arg(0) { - Value::Int(i) => i.checked_abs().map(Value::Int).unwrap_or(Value::Null), - v => v.as_f64().map(|x| real(x.abs())).unwrap_or(Value::Null), + Value::Int(i) => i.checked_abs().map(Value::Int).unwrap_or_else(|| self.fail(Fault::Overflow)), + v => number(v).map(|x| real(x.abs())).unwrap_or(Value::Null), }, Func::Round => { + let x = arg(0).into_owned(); + if blank_text(&x) { + missing.set(true); + } let places = if args.len() > 1 { n(1).map(|p| p.clamp(-15.0, 15.0) as i32) } else { Some(0) }; - match (arg(0).into_owned(), places) { + match (x, places) { (Value::Int(i), Some(p)) if p >= 0 => Value::Int(i), - (v, Some(p)) => match v.as_f64() { + (v, Some(p)) => match number(&v) { Some(x) => { let m = 10f64.powi(p); // So large it has no places to round: as it is. @@ -1793,15 +2016,15 @@ impl<'a> Eval<'a> { } Func::Floor => n(0).map(|x| whole(x.floor())).unwrap_or(Value::Null), Func::Ceil => n(0).map(|x| whole(x.ceil())).unwrap_or(Value::Null), - Func::Sqrt => return num1(f64::sqrt), + Func::Sqrt => return num_from(f64::sqrt, true, Fault::NegativeRoot), Func::Exp => return num1(f64::exp), - Func::Ln => return num1(f64::ln), - Func::Log10 => return num1(f64::log10), + Func::Ln => return num_from(f64::ln, false, Fault::Logarithm), + Func::Log10 => return num_from(f64::log10, false, Fault::Logarithm), Func::Pi => Value::Float(std::f64::consts::PI), Func::Min | Func::Max => { let mut best: Option<(f64, Cow<'a, Value>)> = None; for a in args { - let v = self.value(a); + let v = read(a); let Some(x) = v.as_f64() else { continue }; let better = match &best { None => true, @@ -1816,7 +2039,7 @@ impl<'a> Eval<'a> { Func::Clamp => { // Each argument once: evaluated again, nested clamps would cost 2^depth. let (lo, x, hi) = (arg(0), arg(1), arg(2)); - match (lo.as_f64(), x.as_f64(), hi.as_f64()) { + match (number(&lo), number(&x), number(&hi)) { (Some(l), Some(v), Some(h)) if l <= h => { return if v < l { lo @@ -1826,6 +2049,7 @@ impl<'a> Eval<'a> { x }; } + (Some(_), Some(_), Some(_)) => self.fail(Fault::ClampBounds), _ => Value::Null, } } @@ -1834,7 +2058,11 @@ impl<'a> Eval<'a> { match (n(0), n(1), n(2), n(3), n(4), exponent) { (Some(x), Some(d0), Some(d1), Some(r0), Some(r1), Some(e)) => { let t = if d1 == d0 { if x <= d0 { 0.0 } else { 1.0 } } else { ((x - d0) / (d1 - d0)).clamp(0.0, 1.0) }; - real(r0 + (r1 - r0) * t.powf(e)) + // As `0 ^ -1`: a negative exponent at the domain's start divides by zero. + if t == 0.0 && e < 0.0 { + return Cow::Owned(self.fail(Fault::DivisionByZero)); + } + self.or_fail(checked(r0 + (r1 - r0) * t.powf(e))) } _ => Value::Null, } @@ -1842,7 +2070,7 @@ impl<'a> Eval<'a> { Func::ToString => txt(0).map(text).unwrap_or(Value::Null), Func::ToInt => match &*arg(0) { Value::Int(i) => Value::Int(*i), - v => v.as_f64().map(|x| whole(x.trunc())).unwrap_or(Value::Null), + v => number(v).map(|x| whole(x.trunc())).unwrap_or(Value::Null), }, Func::ToReal => n(0).map(Value::Float).unwrap_or(Value::Null), Func::Length => txt(0).map(|s| Value::Int(s.chars().count() as i64)).unwrap_or(Value::Null), diff --git a/crates/data/src/query/tests.rs b/crates/data/src/query/tests.rs index 70e8705707..3340458f2f 100644 --- a/crates/data/src/query/tests.rs +++ b/crates/data/src/query/tests.rs @@ -773,3 +773,92 @@ fn the_user_guide_lists_what_the_catalogs_hold() { assert!(grammar.contains(&format!("\"{}\"", k.word())), "the grammar should use \"{}\"", k.word()); } } + +/// Each error that gives null says which it was; a null read from a field, written, or left by +/// a guarded branch, and an error whose null a later step replaced, say nothing. +#[test] +fn errors_say_why_they_are_null() { + let d = ds(); + let why = |text: &str, row: usize| { + let e = parse(text, &d).unwrap_or_else(|e| panic!("{text}: {e}")); + value_or_fault(&e, &d, row, &Env::default()) + }; + for (text, fault) in [ + ("pop / 0", Fault::DivisionByZero), + ("pop % 0", Fault::DivisionByZero), + ("1.5 % 0", Fault::DivisionByZero), + ("0 ^ -1", Fault::DivisionByZero), + ("sqrt(0 - pop)", Fault::NegativeRoot), + ("ln(0)", Fault::Logarithm), + ("log10(0 - 1)", Fault::Logarithm), + ("exp(1000)", Fault::Overflow), + ("10 ^ 400", Fault::Overflow), + ("(0 - 8) ^ 0.5", Fault::NotANumber), + ("name * 2", Fault::Text), + ("sqrt(name)", Fault::Text), + ("to_real(name)", Fault::Text), + ("clamp(10, pop, 0)", Fault::ClampBounds), + ("scale_exp(0, 0, 10, 1, 2, -1)", Fault::DivisionByZero), + ("round(pop / 0) + 1", Fault::DivisionByZero), + // An error's null in either position of an operator still reports. + ("(1 / 0) * 2", Fault::DivisionByZero), + ("2 * (1 / 0)", Fault::DivisionByZero), + ("1 / 0 * 2", Fault::DivisionByZero), + ("sqrt(1 / 0)", Fault::DivisionByZero), + ("round(1 / 0, 0)", Fault::DivisionByZero), + ("nullif(pop / 0, 1)", Fault::DivisionByZero), + ("'(' || (pop / 0) || ')'", Fault::DivisionByZero), + ] { + assert_eq!(why(text, 0), (Value::Null, Some(fault)), "{text}"); + } + // A function that skips nulls has each null argument as a cause: fixing the error alone gives + // a value, so the error is said even beside a missing argument. + for text in ["coalesce(1 / 0, pop)", "max(1 / 0, pop)", "min(pop, sqrt(-1))"] { + assert!(why(text, 3).1.is_some(), "{text}"); + } + // Null in, null out, without an error; what an expression asks for or guards against. + for (text, row) in [ + ("pop / 2", 3), + ("null", 0), + ("if(pop > 0, 1)", 3), + ("if(pop / 0 > 1, 1, null)", 0), + ("CASE WHEN pop / 0 > 1 THEN 1 END", 0), + ("CASE pop WHEN 1 / 0 THEN 1 END", 0), + ("nullif(pop, pop)", 0), + // The division's null was replaced; the result is null because `pop` is. + ("coalesce(1 / 0, 0) / pop", 3), + // An error that only decided a condition, a CASE's arm or what nullif compares with. + ("if((1 / 0 IS NULL) AND NULL, 1)", 0), + ("CASE 1 / 0 WHEN 1 THEN 2 ELSE NULL END", 0), + ("nullif(NULL, 1 / 0)", 0), + // A missing operand makes an operator's result null whatever else failed. + ("1 / 0 + pop", 3), + ("pop * (1 / 0)", 3), + ("(1 / 0) || pop", 3), + ("clamp(pop, 1 / 0, 5)", 3), + // Empty or `` text is missing, not text that isn't a number. + ("'' * 2", 0), + ("'' * (1 / 0)", 0), + ("(1 / 0) - ''", 0), + ("sqrt('')", 0), + ("clamp('', 1 / 0, 5)", 0), + ("'' + '' - (1 / 0)", 0), + ("round('', 1 / 0)", 0), + ] { + assert_eq!(why(text, row), (Value::Null, None), "{text}"); + } + assert_eq!(why("coalesce(pop / 0, 7)", 0), (Value::Int(7), None)); + // The arm a CASE takes keeps its own error. + assert_eq!(why("CASE 1 / 0 WHEN 1 THEN 2 ELSE sqrt(-1) END", 0), (Value::Null, Some(Fault::NegativeRoot))); + assert_eq!(why("(1 / 0 IS NULL) AND NULL", 0), (Value::Null, None)); + assert_eq!(arith(Arith::Mod, &Value::Int(i64::MIN), &Value::Int(-1)), Ok(Value::Int(0))); + // A stored infinity or NaN isn't joined as text: `+` on it is null, and says why. + let mut odd = Dataset::new("odd", GeometryType::None, vec![Field::new("x", FieldType::Double)]); + odd.push(None, vec![Value::Float(f64::INFINITY)]); + odd.push(None, vec![Value::Float(f64::NAN)]); + for (text, row) in [("x + 1", 0), ("x + 1", 1), ("'pop ' + x", 0)] { + let e = parse(text, &odd).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(value_or_fault(&e, &odd, row, &Env::default()), (Value::Null, Some(Fault::NotANumber)), "{text}, row {row}"); + } + assert_eq!(why("sqrt(0)", 0), (Value::Float(0.0), None)); +} diff --git a/crates/engine/src/cmd/design.rs b/crates/engine/src/cmd/design.rs index 35edd1cff2..f82365cf2c 100644 --- a/crates/engine/src/cmd/design.rs +++ b/crates/engine/src/cmd/design.rs @@ -214,8 +214,9 @@ pub(crate) fn apply_with>(s: &mut Session, id: u32, changes: let rows: Vec = changes.iter().filter_map(|c| c.row).collect(); // What each calculation rule's whole-number field cut: the rule and the field's name. Each // rule's cuts are said, even when a later rule writes the same field: it may read them. - // With them, per rule, how many results its field couldn't hold (written as null), and its type. - let mut cuts: Vec<(String, String, super::edit::Cuts, usize, FieldType)> = Vec::new(); + // With them, per rule, the results its field couldn't hold and those an error made null + // (`WriteLosses`), and its type. + let mut cuts: Vec<(String, String, super::edit::WriteLosses, FieldType)> = Vec::new(); // Parsing against the fields separates a measurement failure from a stale rule. The latter // may still be skipped below, but an ambiguous area must never disable a valid rule. let metadata = rustgis_data::Dataset::new(l.dataset.name.clone(), l.dataset.geometry_type, l.dataset.fields.clone()); @@ -243,22 +244,17 @@ pub(crate) fn apply_with>(s: &mut Session, id: u32, changes: // A rule that no longer reads (its field was removed, or a hand-edited file) is skipped // rather than blocking every edit. let Ok(calc) = super::edit::Calculation::parse(&rule.expression, &metadata) else { continue }; - let mut cut = super::edit::Cuts::default(); - let mut unread = 0usize; + // Counted as Calculate Field counts them. + // A change set may repeat a row: its losses are counted once. + let mut losses = super::edit::WriteLosses::once_per_row(); for r in &rows { - let result = calc.eval(&l.dataset, *r); - let asked_null = result.asks_null(); - let (v, c) = super::edit::coerce_cut(result, ty); - // A result the field can't hold (a date in a number field) is null: counted, as - // Calculate Field counts it. - unread += usize::from(!asked_null && v.is_null() && c == super::edit::Cut::None); - cut.add(c); + let v = losses.write(*r, calc.eval_or_fault(&l.dataset, *r), ty); if let Some(cell) = l.dataset.features.get_mut(*r).and_then(|f| f.attrs.get_mut(fi)) { *cell = v; } } let name = l.dataset.fields.get(fi).map(|f| f.name.clone()).unwrap_or_default(); - cuts.push((rule.name.clone(), name, cut, unread, ty)); + cuts.push((rule.name.clone(), name, losses, ty)); } // Validation. Domains and contingent values are checked on the fields an edit changed (all // of a new feature's), so a value set before the domain doesn't block unrelated edits. @@ -343,13 +339,10 @@ pub(crate) fn apply_with>(s: &mut Session, id: u32, changes: e.modified += counts.modified; e.deleted += counts.deleted; // Said once the edit is kept: a rejected one changed nothing. - for (rule, field, cut, unread, ty) in cuts { - for notice in cut.notices(&format!("Attribute rule {rule}"), &field) { - s.notify(notice, false); - } - if unread > 0 { - let rows = if unread == 1 { "1 row".to_string() } else { format!("{unread} rows") }; - s.notify(format!("Attribute rule {rule}: '{field}' is null in {rows}, whose result isn't a {} value", ty.label()), false); + for (rule, field, losses, ty) in cuts { + let by = crate::notice_text::NoticeText::format("Attribute rule {rule}", [("rule", rule.into())]); + for notice in losses.notices(by, &field, ty) { + s.notify_text(notice, false); } } Ok(()) diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index ed5bc33a19..e5c6ab6bb1 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -6,6 +6,7 @@ use serde_json::{Value as Json, json}; use std::borrow::Cow; use super::{need_layer_or_table, ps}; +use crate::notice_text::NoticeText; use crate::{CmdResult, CommandSpec as C, Session}; pub fn commands() -> Vec { @@ -356,7 +357,7 @@ pub(crate) fn coerce(v: Value, ty: FieldType) -> Value { /// What [`coerce_cut`] took from a number to put it in a whole-number field, or from a date-time /// to put it in a date field. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub(crate) enum Cut { /// Nothing. None, @@ -414,6 +415,105 @@ impl Cuts { } } +/// The rows a notice lists by number, at most. +const LISTED_ROWS: usize = 3; + +/// What writing a calculation's results into one field lost, for its notices: what the field cut +/// ([`Cuts`]), results it can't hold (text that isn't a number or a date), and nulls an +/// expression's error gave (a division by zero), by kind with the first rows of each. +#[derive(Clone, Debug, Default)] +pub(crate) struct WriteLosses { + cuts: Cuts, + unread: usize, + /// Each kind of error met, in the order met: how many rows, and the first [`LISTED_ROWS`]. + errors: Vec<(rustgis_data::query::Fault, usize, Vec)>, + /// Each row and kind of loss counted, for an edit that may write a row more than once + /// ([`once_per_row`]); none where each row is written once (Calculate Field), so nothing + /// grows with the rows. + /// + /// [`once_per_row`]: WriteLosses::once_per_row + counted: Option>, +} + +/// A kind of loss [`WriteLosses`] counts. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum Loss { + Cut(Cut), + Unread, + Error(rustgis_data::query::Fault), +} + +impl WriteLosses { + /// A tally for an edit that may write the same row more than once, counting each kind of + /// loss once per row: a later write that loses something else is still counted. + pub(crate) fn once_per_row() -> WriteLosses { + WriteLosses { counted: Some(Default::default()), ..Default::default() } + } + + /// Whether `row`'s `loss` is yet to be counted, marking it counted. + fn first(&mut self, row: usize, loss: Loss) -> bool { + self.counted.as_mut().is_none_or(|c| c.insert((row, loss))) + } + + /// Row `row`'s result and the error that made it null, if one did, as a `ty` field holds it, + /// counting what that lost. + pub(crate) fn write(&mut self, row: usize, (v, fault): (Value, Option), ty: FieldType) -> Value { + let asked_null = v.asks_null(); + let (v, cut) = coerce_cut(v, ty); + if cut != Cut::None && self.first(row, Loss::Cut(cut)) { + self.cuts.add(cut); + } + if let Some(fault) = fault.filter(|_| v.is_null()) { + if !self.first(row, Loss::Error(fault)) { + return v; + } + match self.errors.iter_mut().find(|(f, ..)| *f == fault) { + Some((_, n, rows)) => { + *n += 1; + if rows.len() < LISTED_ROWS { + rows.push(row); + } + } + None => self.errors.push((fault, 1, vec![row])), + } + } else if !asked_null && v.is_null() && cut == Cut::None && self.first(row, Loss::Unread) { + self.unread += 1; + } + v + } + + /// The notices for what writing `field`, of type `ty`, lost, each starting with `by` (the + /// command or rule that wrote it). Rows are numbered from 1, as `@row_number` numbers them. + /// The error notices are templates a translated interface shows in its language. + pub(crate) fn notices(&self, by: NoticeText, field: &str, ty: FieldType) -> Vec { + let rows = |n: usize| if n == 1 { "1 row".to_string() } else { format!("{n} rows") }; + let english = by.english(); + let mut out: Vec = self.cuts.notices(&english, field).into_iter().map(NoticeText::from).collect(); + if self.unread > 0 { + out.push(format!("{english}: '{field}' is null in {}, whose result isn't a {} value", rows(self.unread), ty.label()).into()); + } + for (fault, n, listed) in &self.errors { + let numbers: Vec = listed.iter().map(|r| r.saturating_add(1).to_string()).collect(); + let key = match (*n, *n > listed.len()) { + (1, _) => "{by}: '{field}' is null in 1 row because of {cause} (row {rows})", + (_, false) => "{by}: '{field}' is null in {count} rows because of {cause} (rows {rows})", + (_, true) => "{by}: '{field}' is null in {count} rows because of {cause} (rows {rows}…)", + }; + out.push(NoticeText::format( + key, + [ + ("by", by.clone()), + ("field", field.to_string().into()), + ("count", n.to_string().into()), + ("cause", NoticeText::term(fault.describe())), + ("rows", numbers.join(", ").into()), + ], + )); + } + out + } +} + /// [`coerce`], with a number past the 64-bit range null, and what it took from a number to put it /// in a whole-number field. pub(crate) fn coerce_cut(v: Value, ty: FieldType) -> (Value, Cut) { @@ -1322,12 +1422,12 @@ fn land_table_edit(s: &mut Session, p: &Json, command: &str) -> Option Result<(Dataset, Cuts, usize), String> { +fn calculate_table(input: &Dataset, p: &Json) -> Result<(Dataset, WriteLosses), String> { let fi = input.field_index(ps(p, "field").ok_or("field is required")?).ok_or("no field")?; let ty = input.fields.get(fi).map(|f| f.ty).ok_or("no field")?; let expr = ps(p, "expression").map(str::trim).filter(|e| !e.is_empty()).ok_or("expression is required")?; let calc = Calculation::parse(expr, input)?; - let (out, cuts, unread) = evaluate(&calc, input, (0..input.features.len()).collect(), ty)?; + let (out, losses) = evaluate(&calc, input, (0..input.features.len()).collect(), ty)?; let mut result = input.clone(); rustgis_data::progress::begin(out.len()); for (r, v) in out { @@ -1336,7 +1436,7 @@ fn calculate_table(input: &Dataset, p: &Json) -> Result<(Dataset, Cuts, usize), *c = v; } } - Ok((result, cuts, unread)) + Ok((result, losses)) } /// Sets a field from an expression, on the selected rows or all of them (all of a standalone @@ -1354,10 +1454,10 @@ fn calculate_field(s: &mut Session, p: &Json) -> CmdResult { let Target::Layer(id) = t else { editing_on(s)?; let (ty, field) = data(s, t)?.fields.get(fi).map(|f| (f.ty, f.name.clone())).ok_or("no field")?; - let (result, cuts, unread) = calculate_table(data(s, t)?, p)?; + let (result, losses) = calculate_table(data(s, t)?, p)?; let n = result.features.len(); data_mut(s, t)?.features = result.features; - calculation_notices(s, &field, ty, cuts, unread); + calculation_notices(s, &field, ty, &losses); return Ok(json!({"rows": n})); }; let l = s.map_mut().layer_mut(id).ok_or("no layer")?; @@ -1365,7 +1465,7 @@ fn calculate_field(s: &mut Session, p: &Json) -> CmdResult { let calc = Calculation::parse(&expr, &l.dataset)?; let rows: Vec = if l.selection.is_empty() { (0..l.dataset.features.len()).collect() } else { l.selection.clone() }; let n = rows.len(); - let (out, cuts, unread) = evaluate(&calc, &l.dataset, rows, ty)?; + let (out, losses) = evaluate(&calc, &l.dataset, rows, ty)?; let mut changes = Vec::with_capacity(out.len()); let original = l.dataset.features.clone(); let archiving = l.design.archive.is_some(); @@ -1384,38 +1484,28 @@ fn calculate_field(s: &mut Session, p: &Json) -> CmdResult { } super::design::apply_with(s, id, changes, false)?; let field = s.map().layer(id).and_then(|l| l.dataset.fields.get(fi)).map(|f| f.name.clone()).unwrap_or_default(); - calculation_notices(s, &field, ty, cuts, unread); + calculation_notices(s, &field, ty, &losses); Ok(json!({"rows": n})) } -/// `calc` on `rows` of `ds`, each result as a `ty` field holds it: the rows' new values, the -/// results a whole-number or date field cut, and how many other results it can't hold (text -/// that isn't a number or a date), which are null. -fn evaluate(calc: &Calculation, ds: &Dataset, rows: Vec, ty: FieldType) -> Result<(Vec<(usize, Value)>, Cuts, usize), String> { +/// `calc` on `rows` of `ds`, each result as a `ty` field holds it: the rows' new values, and what +/// writing them loses ([`WriteLosses`]). +fn evaluate(calc: &Calculation, ds: &Dataset, rows: Vec, ty: FieldType) -> Result<(Vec<(usize, Value)>, WriteLosses), String> { let mut out = Vec::with_capacity(rows.len()); - let (mut cuts, mut unread) = (Cuts::default(), 0usize); + let mut losses = WriteLosses::default(); rustgis_data::progress::begin(rows.len()); for r in rows { rustgis_data::progress::step(1).map_err(|e| e.to_string())?; - let v = calc.eval(ds, r); - let asked_null = v.asks_null(); - let (v, cut) = coerce_cut(v, ty); - unread += usize::from(!asked_null && v.is_null() && cut == Cut::None); - cuts.add(cut); - out.push((r, v)); + out.push((r, losses.write(r, calc.eval_or_fault(ds, r), ty))); } - Ok((out, cuts, unread)) + Ok((out, losses)) } /// Says what Calculate Field's results lost in `field` ([`evaluate`]), even when a calculation /// rule writes the field afterwards: the rule may read the cut value, and says what it cuts itself. -fn calculation_notices(s: &mut Session, field: &str, ty: FieldType, cuts: Cuts, unread: usize) { - for notice in cuts.notices("Calculate Field", field) { - s.notify(notice, false); - } - if unread > 0 { - let rows = if unread == 1 { "1 row" } else { &format!("{unread} rows") }; - s.notify(format!("Calculate Field: '{field}' is null in {rows}, whose result isn't a {} value", ty.label()), false); +fn calculation_notices(s: &mut Session, field: &str, ty: FieldType, losses: &WriteLosses) { + for notice in losses.notices(NoticeText::term("Calculate Field"), field, ty) { + s.notify_text(notice, false); } } @@ -1467,10 +1557,16 @@ impl Calculation { } } + #[cfg(test)] pub(crate) fn eval(&self, ds: &rustgis_data::Dataset, r: usize) -> Value { + self.eval_or_fault(ds, r).0 + } + + /// Row `r`'s result, with the error that made it null, if one did. + pub(crate) fn eval_or_fault(&self, ds: &rustgis_data::Dataset, r: usize) -> (Value, Option) { match self { - Calculation::Expr(e) => rustgis_data::query::value(e, ds, r, &rustgis_data::query::Env::default()), - Calculation::Calc(c) => c.eval(ds, r), + Calculation::Expr(e) => rustgis_data::query::value_or_fault(e, ds, r, &rustgis_data::query::Env::default()), + Calculation::Calc(c) => c.eval_or_fault(ds, r), } } } @@ -1508,7 +1604,15 @@ impl Calc { Ok(Calc { a: term(lhs)?, op, b: rhs.map(term).transpose()? }) } + #[cfg(test)] pub(crate) fn eval(&self, ds: &rustgis_data::Dataset, r: usize) -> Value { + self.eval_or_fault(ds, r).0 + } + + /// Row `r`'s result, with the error that made it null: dividing by zero, or `-`, `*`, `/` on + /// a value that isn't a number. + pub(crate) fn eval_or_fault(&self, ds: &rustgis_data::Dataset, r: usize) -> (Value, Option) { + use rustgis_data::query::Fault; let get = |t: &Term| -> Value { match t { Term::Num(n) => Value::Float(*n), @@ -1516,20 +1620,36 @@ impl Calc { Term::Field(i) => ds.value(r, *i).clone(), } }; + // A number result: past the largest number a value holds, it is null, and says why. + let number = |f: f64| { + if f.is_finite() { + (Value::Float(f), None) + } else { + (Value::Null, Some(if f.is_nan() { Fault::NotANumber } else { Fault::Overflow })) + } + }; let va = get(&self.a); match (&self.b, self.op) { (Some(b), Some(op)) => { let vb = get(b); match (va.as_f64(), vb.as_f64(), op) { - (Some(x), Some(y), '+') => Value::Float(x + y), - (Some(x), Some(y), '-') => Value::Float(x - y), - (Some(x), Some(y), '*') => Value::Float(x * y), - (Some(x), Some(y), '/') if y != 0.0 => Value::Float(x / y), - (_, _, '+') if !va.is_null() && !vb.is_null() => Value::Text(format!("{va}{vb}")), - _ => Value::Null, + (Some(x), Some(y), '+') => number(x + y), + (Some(x), Some(y), '-') => number(x - y), + (Some(x), Some(y), '*') => number(x * y), + (Some(_), Some(0.0), '/') => (Value::Null, Some(Fault::DivisionByZero)), + (Some(x), Some(y), '/') => number(x / y), + // A stored NaN or infinity isn't text to join. + (_, _, '+') if !va.is_null() && !vb.is_null() && !va.is_non_finite() && !vb.is_non_finite() => { + (Value::Text(format!("{va}{vb}")), None) + } + // Empty or `` text is missing to `-`, `*`, `/` (`+` joins it, above). + _ if va.is_null() || vb.is_null() || (op != '+' && (va.asks_null() || vb.asks_null())) => (Value::Null, None), + // `+` joins text; what it couldn't join is a stored NaN or infinity. + (_, _, '+') => (Value::Null, Some(Fault::NotANumber)), + _ => (Value::Null, Fault::of_non_number(&va).or(Fault::of_non_number(&vb))), } } - _ => va, + _ => (va, None), } } } @@ -1557,6 +1677,64 @@ mod tests { use super::*; use rustgis_data::{Dataset, Field}; + /// A null an expression's error wrote says so, by kind with its count and first rows, on a + /// layer and a table, from the language and from the earlier calculator; a clean run, and + /// nulls read from the data, say nothing. + #[test] + fn calculation_errors_are_said() { + for table in [false, true] { + let mut s = Session::new(crate::Project::default()); + let fields = ["n", "Pop Est", "r"].map(|f| Field::new(f, FieldType::Double)).to_vec(); + let mut ds = Dataset::new("calc", GeometryType::None, fields); + for n in [Value::Float(2.0), Value::Float(0.0), Value::Null, Value::Float(0.0), Value::Float(0.0), Value::Float(0.0)] { + ds.push(None, vec![n, Value::Float(5.0), Value::Null]); + } + let target = if table { + json!({"table": s.project.add_table(ds)}) + } else { + let id = s.map_mut().add_dataset(ds); + json!({"layer": id}) + }; + for (expression, said) in [ + ("1 / n", vec!["Calculate Field: 'r' is null in 4 rows because of a division by zero (rows 2, 4, 5…)"]), + ("Pop Est / n", vec!["Calculate Field: 'r' is null in 4 rows because of a division by zero (rows 2, 4, 5…)"]), + ( + "if(n = 2, sqrt(0 - n), n * 2)", + vec!["Calculate Field: 'r' is null in 1 row because of the square root of a negative number (row 1)"], + ), + ("n * 2", vec![]), + ] { + let mut p = target.clone(); + p["field"] = json!("r"); + p["expression"] = json!(expression); + let mark = s.notice_mark(); + s.run("edit.calculateField", &p).unwrap(); + let notices: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert_eq!(notices, said, "{expression}, table {table}"); + } + } + // The earlier calculator doesn't join a stored infinity as text either. + let mut ds = Dataset::new("odd", GeometryType::None, vec![Field::new("Pop Est", FieldType::Double)]); + ds.push(None, vec![Value::Float(f64::INFINITY)]); + let calc = Calc::parse("Pop Est + 1", &ds).unwrap(); + assert_eq!(calc.eval_or_fault(&ds, 0), (Value::Null, Some(rustgis_data::query::Fault::NotANumber))); + // Nor stores one it overflows to: it is null, too large. + ds.push(None, vec![Value::Float(1e308)]); + // A row an edit writes twice is counted once. + // A later write's different loss is still counted. + let mut losses = WriteLosses::once_per_row(); + losses.write(4, (Value::Float(1.0), None), FieldType::Double); + for row in [4, 4] { + losses.write(row, (Value::Null, Some(rustgis_data::query::Fault::DivisionByZero)), FieldType::Double); + } + let said: Vec = losses.notices(NoticeText::term("Rule"), "n", FieldType::Double).iter().map(NoticeText::english).collect(); + assert_eq!(said, ["Rule: 'n' is null in 1 row because of a division by zero (row 5)"]); + let calc = Calc::parse("'pop ' + Pop Est", &ds).unwrap(); + assert_eq!(calc.eval_or_fault(&ds, 0), (Value::Null, Some(rustgis_data::query::Fault::NotANumber))); + let calc = Calc::parse("Pop Est * 10", &ds).unwrap(); + assert_eq!(calc.eval_or_fault(&ds, 1), (Value::Null, Some(rustgis_data::query::Fault::Overflow))); + } + /// Calculate Field never adds fields: layer and table callers get the same next step. #[test] fn missing_calculation_field_points_to_add_field() { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4e02e3f5df..9aa9042897 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1098,6 +1098,17 @@ fn date_fields_work_from_adding_to_saving() { assert_eq!(notices.contains(&null), said, "{expression}: {notices:?}"); s.run("data.attributeRule", &json!({"layer": "Cities", "name": "Year", "remove": true})).unwrap(); } + // A rule whose expression fails writes null, and says why. + let rule = json!({"layer": "Cities", "name": "Year", "kind": "calculation", "field": "yr", "expression": "1 / 0"}); + s.run("data.attributeRule", &rule).unwrap(); + let mark = s.notice_mark(); + s.run("edit.setValue", &json!({"layer": "Cities", "row": 0, "field": "founded", "value": "2026-03-05"})).unwrap(); + let notices: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!( + notices.contains(&"Attribute rule Year: 'yr' is null in 1 row because of a division by zero (row 1)".to_string()), + "{notices:?}" + ); + s.run("data.attributeRule", &json!({"layer": "Cities", "name": "Year", "remove": true})).unwrap(); s.run("edit.deleteField", &json!({"layer": "Cities", "field": "yr"})).unwrap(); // Saved and opened, the values are still dates. // Removed however the test ends, and its own each run (the process id alone can repeat). diff --git a/docs/user-guide/expressions.md b/docs/user-guide/expressions.md index 4377984335..8bf1564afd 100644 --- a/docs/user-guide/expressions.md +++ b/docs/user-guide/expressions.md @@ -132,7 +132,8 @@ Rules shared by all of them: matches neither `pop > 100` nor `NOT pop > 100`; `IS NULL` tests for it. `AND` is false when either side is false and `OR` true when either side is true, whatever the other side is. - **Errors give null**: a division by zero, the square root of a negative number, a result too - large to represent. + large to represent. Where the result is stored (Calculate Field, calculation attribute rules), + a notice says how many rows an error made null, which error, and the first three of those rows. - **`IN`** compares with each value as `=` does; a `NULL` in the list matches nothing, so `x NOT IN (1, NULL)` is true for 2. - **As a condition**, a value that isn't true or false counts as true when it is a non-zero