diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ceaee..552e71b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,13 @@ part was built. - **The image hash and the PNG**: `sha256:` and the lowercase hex SHA-256 of the raw RGBA8 bytes (ADR 0009), with Lumen's own SHA-256; a PNG encoder with stored deflate blocks, whose bytes are a fixed function of the pixels (`write_png`, `png_len`). (flyingrobots/lumen#2.) +- **A JSON reader that keeps numbers as text** (`parse_json`, `JsonValue`, `JsonLimits`, + `JsonError`, `JSON_READER_ID`; [reference](docs/reference/library.md#the-json-reader)): bytes + in, a value tree out, with no float anywhere. A number is its source text, checked against the + JSON grammar and returned as written, so a later layer converts a decimal exactly. Duplicate + keys, invalid UTF-8, lone surrogates, trailing content, nesting past a depth limit and input + past a size limit are refused, each with a line and a character column. Nothing uses it yet. + (flyingrobots/lumen#211.) - **Tests**: hand-derived radiance, shadows against exact geometry, every ray and every sample against brute force and an independent composition, order independence, counts against oracles other than the counting code, golden image hashes and content digests (one value for diff --git a/crates/lumen/src/json.rs b/crates/lumen/src/json.rs new file mode 100644 index 0000000..15783d6 --- /dev/null +++ b/crates/lumen/src/json.rs @@ -0,0 +1,57 @@ +//! A JSON reader that keeps numbers as text (`lumen-json-reader/1`, flyingrobots/lumen#211). +//! +//! [`parse_json`] reads a document from bytes into a [`JsonValue`] tree. It exists so that job +//! files (ADR 0015) and, later, scene files are read by Lumen's own code (R-DEP-1: no third-party +//! crate), and it contains no arithmetic and no float. A number is not converted: the tree holds +//! the token as written, after it has been checked against the grammar of RFC 8259, so a later +//! layer converts a decimal exactly (Bunny's `bunny-decimal-q32-32/1`) and never through a float. +//! The reader reads no file and asks the host nothing; the caller passes the bytes. +//! +//! What it decides, for a document that is valid JSON by RFC 8259: +//! +//! - The input is bytes and must be UTF-8. A byte order mark, a UTF-16 document and any invalid +//! sequence are refused. `\u` escapes are decoded, a surrogate pair into one character, and a +//! lone surrogate is refused. +//! - A duplicate key in one object is refused (RFC 8259 leaves it open). Keys are compared after +//! their escapes are decoded. An object keeps its members in document order. +//! - A number is `-? (0 | [1-9][0-9]*) (. [0-9]+)? ([eE] [+-]? [0-9]+)?` and nothing else. Its +//! size and magnitude are not limited beyond the document's size: `1E400` and a 1 000-digit +//! integer are tokens like any other, and `-0` stays `-0`. +//! - The document is one value with optional white space (space, tab, line feed, carriage +//! return) around it; anything after it is refused. +//! - Limits are parameters ([`JsonLimits`]). A document longer than `max_bytes` is refused +//! before it is read, and a container that would nest deeper than `max_depth` is refused +//! before anything is stored for it. Depth counts open containers, so `max_depth` 2 admits +//! `[[]]` and refuses `[[[]]]`; a scalar has depth 0. The reader does not recurse, and the +//! depth limit has a ceiling ([`JSON_MAX_DEPTH_CEILING`]) because dropping or comparing a deep +//! tree does recurse. +//! - Every error carries a byte offset, a 1-based line and a 1-based column ([`JsonError`]). A +//! column counts characters (Unicode scalar values), not bytes; a line ends at each line feed +//! (a carriage return is a character of its line). +//! - No input panics: every index is checked and every arithmetic step is bounded by the +//! document's length. + +mod error; +mod limits; +mod parser; +mod scan; +mod value; + +#[cfg(test)] +mod conformance; +#[cfg(test)] +mod generated; +#[cfg(test)] +mod tests; + +pub use error::{JsonError, JsonErrorKind}; +pub use limits::{ + JsonLimitError, JsonLimits, JSON_DEFAULT_MAX_BYTES, JSON_DEFAULT_MAX_DEPTH, + JSON_MAX_DEPTH_CEILING, +}; +pub use parser::parse_json; +pub use value::JsonValue; + +/// The identity of this reader's contract: the grammar, the UTF-8 rule, the refusals above and +/// the position rule. A change to any of them is a new identity. +pub const JSON_READER_ID: &str = "lumen-json-reader/1"; diff --git a/crates/lumen/src/json/conformance.rs b/crates/lumen/src/json/conformance.rs new file mode 100644 index 0000000..179adb7 --- /dev/null +++ b/crates/lumen/src/json/conformance.rs @@ -0,0 +1 @@ +//! placeholder diff --git a/crates/lumen/src/json/error.rs b/crates/lumen/src/json/error.rs new file mode 100644 index 0000000..7e60dc0 --- /dev/null +++ b/crates/lumen/src/json/error.rs @@ -0,0 +1,139 @@ +//! Why a document was refused, and where: [`JsonError`]. + +use core::fmt; + +/// What was wrong with a document. +/// +/// Non-exhaustive: a later version may report a new failure, so a `match` outside this crate +/// needs a wildcard arm. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum JsonErrorKind { + /// The input is longer than the size limit. The position is that of the first byte past it; + /// nothing was read. + TooLarge { + /// The input's length in bytes. + length: usize, + /// The limit in bytes. + limit: usize, + }, + /// A container opens deeper than the depth limit. The position is its opening bracket. + TooDeep { + /// The limit. + limit: usize, + }, + /// The input ended where more was needed. + UnexpectedEnd { + /// What was needed. + expected: &'static str, + }, + /// A byte that cannot come here. + UnexpectedByte { + /// The byte. + byte: u8, + /// What could come instead. + expected: &'static str, + }, + /// A control character (below U+0020) inside a string, which must be escaped. + ControlCharacter { + /// The byte. + byte: u8, + }, + /// Bytes inside a string that are not UTF-8. The position is the first bad byte. + InvalidUtf8, + /// A `\u` escape of a surrogate that is not half of a pair. The position is the escape's + /// backslash (the first one, for a high surrogate not followed by a low one). + LoneSurrogate { + /// The escape's code unit. + unit: u16, + }, + /// A key that an object already has. The position is the second key's opening quote. + DuplicateKey { + /// The key, decoded. + key: String, + }, +} + +/// A refused document: what was wrong ([`JsonError::kind`]) and where. +/// +/// The position is a byte offset, a line and a column, all pointing at the same place. Lines +/// and columns start at 1; a line ends at each line feed; a column counts characters (Unicode +/// scalar values) from the start of its line, so a character of several bytes is one column. +/// The offset of an error at the end of the input is the input's length. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JsonError { + kind: JsonErrorKind, + offset: usize, + line: usize, + column: usize, +} + +impl JsonError { + /// An error of `kind` at byte `offset` of `bytes`, with its line and column worked out. + pub(super) fn at(bytes: &[u8], offset: usize, kind: JsonErrorKind) -> Self { + let before = bytes.get(..offset).unwrap_or(bytes); + let line = 1 + before.iter().filter(|&&b| b == b'\n').count(); + let line_start = before.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1); + let column = 1 + before + .get(line_start..) + .unwrap_or_default() + .iter() + .filter(|&&b| b & 0xC0 != 0x80) + .count(); + Self { kind, offset, line, column } + } + + /// What was wrong. + #[must_use] + pub const fn kind(&self) -> &JsonErrorKind { + &self.kind + } + + /// The byte offset, from 0. + #[must_use] + pub const fn offset(&self) -> usize { + self.offset + } + + /// The line, from 1. + #[must_use] + pub const fn line(&self) -> usize { + self.line + } + + /// The column in characters, from 1. + #[must_use] + pub const fn column(&self) -> usize { + self.column + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "line {}, column {}: ", self.line, self.column)?; + match &self.kind { + JsonErrorKind::TooLarge { length, limit } => { + write!(f, "the input is {length} bytes, over the limit of {limit}") + } + JsonErrorKind::TooDeep { limit } => { + write!(f, "nesting deeper than the limit of {limit}") + } + JsonErrorKind::UnexpectedEnd { expected } => { + write!(f, "the input ends where {expected} was expected") + } + JsonErrorKind::UnexpectedByte { byte, expected } => { + write!(f, "byte 0x{byte:02X} where {expected} was expected") + } + JsonErrorKind::ControlCharacter { byte } => { + write!(f, "control character 0x{byte:02X} in a string; escape it") + } + JsonErrorKind::InvalidUtf8 => write!(f, "bytes that are not UTF-8"), + JsonErrorKind::LoneSurrogate { unit } => { + write!(f, "the escape of surrogate U+{unit:04X} is not half of a pair") + } + JsonErrorKind::DuplicateKey { key } => write!(f, "the key {key:?} appears twice"), + } + } +} + +impl std::error::Error for JsonError {} diff --git a/crates/lumen/src/json/generated.rs b/crates/lumen/src/json/generated.rs new file mode 100644 index 0000000..179adb7 --- /dev/null +++ b/crates/lumen/src/json/generated.rs @@ -0,0 +1 @@ +//! placeholder diff --git a/crates/lumen/src/json/limits.rs b/crates/lumen/src/json/limits.rs new file mode 100644 index 0000000..2e4ead2 --- /dev/null +++ b/crates/lumen/src/json/limits.rs @@ -0,0 +1,94 @@ +//! The reader's limits: [`JsonLimits`]. + +use core::fmt; + +/// The default depth limit: 128 open containers, far past any job or scene file and well inside +/// [`JSON_MAX_DEPTH_CEILING`]. +pub const JSON_DEFAULT_MAX_DEPTH: usize = 128; + +/// The default size limit: 16 MiB (`2^24` bytes) of input. A scene file with its meshes inline +/// would need more; its reader passes its own limit. +pub const JSON_DEFAULT_MAX_BYTES: usize = 1 << 24; + +/// The largest depth limit [`JsonLimits::new`] accepts. The reader itself does not recurse, but +/// the tree's `Drop`, `Clone`, `Debug` and `PartialEq` do, one stack frame per level, so the +/// limit keeps them inside a test thread's 2 MiB stack. +pub const JSON_MAX_DEPTH_CEILING: usize = 1024; + +/// The limits of one read: how deep containers may nest and how long the input may be. +/// +/// # Examples +/// +/// ``` +/// use lumen::{parse_json, JsonLimits}; +/// +/// let limits = JsonLimits::new(2, 64)?; +/// assert!(parse_json(b"[[]]", limits).is_ok()); +/// assert!(parse_json(b"[[[]]]", limits).is_err()); +/// # Ok::<(), lumen::JsonLimitError>(()) +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JsonLimits { + max_depth: usize, + max_bytes: usize, +} + +impl JsonLimits { + /// Limits of `max_depth` open containers (0 admits scalars only) and `max_bytes` bytes of + /// input (the input may be exactly that long). + /// + /// # Errors + /// + /// [`JsonLimitError`] if `max_depth` is above [`JSON_MAX_DEPTH_CEILING`]. + pub const fn new(max_depth: usize, max_bytes: usize) -> Result { + if max_depth > JSON_MAX_DEPTH_CEILING { + return Err(JsonLimitError { max_depth }); + } + Ok(Self { max_depth, max_bytes }) + } + + /// The deepest nesting of containers admitted. + #[must_use] + pub const fn max_depth(self) -> usize { + self.max_depth + } + + /// The longest input admitted, in bytes. + #[must_use] + pub const fn max_bytes(self) -> usize { + self.max_bytes + } +} + +impl Default for JsonLimits { + /// [`JSON_DEFAULT_MAX_DEPTH`] and [`JSON_DEFAULT_MAX_BYTES`]. + fn default() -> Self { + Self { max_depth: JSON_DEFAULT_MAX_DEPTH, max_bytes: JSON_DEFAULT_MAX_BYTES } + } +} + +/// A depth limit above [`JSON_MAX_DEPTH_CEILING`] was asked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JsonLimitError { + max_depth: usize, +} + +impl JsonLimitError { + /// The depth limit that was refused. + #[must_use] + pub const fn max_depth(self) -> usize { + self.max_depth + } +} + +impl fmt::Display for JsonLimitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "a JSON depth limit of {} is above the ceiling of {JSON_MAX_DEPTH_CEILING}", + self.max_depth + ) + } +} + +impl std::error::Error for JsonLimitError {} diff --git a/crates/lumen/src/json/parser.rs b/crates/lumen/src/json/parser.rs new file mode 100644 index 0000000..6fcffd4 --- /dev/null +++ b/crates/lumen/src/json/parser.rs @@ -0,0 +1,156 @@ +//! The reader's grammar level: [`parse_json`], an iterative parser with an explicit stack. + +use std::collections::BTreeSet; + +use super::error::{JsonError, JsonErrorKind}; +use super::limits::JsonLimits; +use super::scan::Scanner; +use super::value::JsonValue; + +/// A container being read. +enum Frame { + Array(Vec), + Object { members: Vec<(String, JsonValue)>, keys: BTreeSet, key: String }, +} + +/// Reads `bytes` as one JSON document under `limits`. +/// +/// The input is checked against the size limit before anything is read, and each container +/// against the depth limit before anything is stored for it. The tree is built with an explicit +/// stack of at most `limits.max_depth()` frames, so no input recurses. See the +/// [module](self) for what is accepted and refused. +/// +/// # Errors +/// +/// A [`JsonError`] with the position of the first fault, in input order. +/// +/// # Examples +/// +/// ``` +/// use lumen::{parse_json, JsonLimits, JsonValue}; +/// +/// let value = parse_json(br#"{"width": 320, "scale": 1.50}"#, JsonLimits::default())?; +/// assert_eq!(value.get("scale"), Some(&JsonValue::Number("1.50".to_owned()))); +/// let error = parse_json(b"[1,\n 2,]", JsonLimits::default()).unwrap_err(); +/// assert_eq!((error.line(), error.column()), (2, 4)); +/// # Ok::<(), lumen::JsonError>(()) +/// ``` +pub fn parse_json(bytes: &[u8], limits: JsonLimits) -> Result { + if bytes.len() > limits.max_bytes() { + let kind = JsonErrorKind::TooLarge { length: bytes.len(), limit: limits.max_bytes() }; + return Err(JsonError::at(bytes, limits.max_bytes(), kind)); + } + let mut scan = Scanner::new(bytes); + let mut stack: Vec = Vec::new(); + loop { + scan.skip_white_space(); + let start = scan.pos; + let mut value = match scan.peek() { + Some(b'[') => { + if stack.len() >= limits.max_depth() { + return scan.error(start, JsonErrorKind::TooDeep { limit: limits.max_depth() }); + } + scan.pos += 1; + scan.skip_white_space(); + if scan.peek() == Some(b']') { + scan.pos += 1; + JsonValue::Array(Vec::new()) + } else { + stack.push(Frame::Array(Vec::new())); + continue; + } + } + Some(b'{') => { + if stack.len() >= limits.max_depth() { + return scan.error(start, JsonErrorKind::TooDeep { limit: limits.max_depth() }); + } + scan.pos += 1; + scan.skip_white_space(); + if scan.peek() == Some(b'}') { + scan.pos += 1; + JsonValue::Object(Vec::new()) + } else { + let mut keys = BTreeSet::new(); + let key = read_key(&mut scan, &mut keys, "a string key or '}'")?; + stack.push(Frame::Object { members: Vec::new(), keys, key }); + continue; + } + } + Some(b'"') => JsonValue::String(scan.string()?), + Some(b't') => scan.word(b"true").map(|()| JsonValue::Bool(true))?, + Some(b'f') => scan.word(b"false").map(|()| JsonValue::Bool(false))?, + Some(b'n') => scan.word(b"null").map(|()| JsonValue::Null)?, + Some(b'-' | b'0'..=b'9') => JsonValue::Number(scan.number()?), + _ => return scan.unexpected("a value"), + }; + // `value` is complete: attach it to the container it ends in, closing containers + // whose last element it was, until one wants another element or none is open. + loop { + let Some(frame) = stack.last_mut() else { + scan.skip_white_space(); + return if scan.pos == bytes.len() { + Ok(value) + } else { + scan.unexpected("the end of the input") + }; + }; + scan.skip_white_space(); + match frame { + Frame::Array(items) => { + items.push(value); + match scan.peek() { + Some(b',') => { + scan.pos += 1; + break; + } + Some(b']') => { + scan.pos += 1; + value = JsonValue::Array(core::mem::take(items)); + } + _ => return scan.unexpected("',' or ']'"), + } + } + Frame::Object { members, keys, key } => { + members.push((core::mem::take(key), value)); + match scan.peek() { + Some(b',') => { + scan.pos += 1; + scan.skip_white_space(); + *key = read_key(&mut scan, keys, "a string key")?; + break; + } + Some(b'}') => { + scan.pos += 1; + value = JsonValue::Object(core::mem::take(members)); + } + _ => return scan.unexpected("',' or '}'"), + } + } + } + stack.pop(); + } + } +} + +/// A member's key and the colon after it, the key's first byte at the cursor. A key already in +/// `keys` is refused at its opening quote. +fn read_key( + scan: &mut Scanner<'_>, + keys: &mut BTreeSet, + expected: &'static str, +) -> Result { + if scan.peek() != Some(b'"') { + return scan.unexpected(expected); + } + let start = scan.pos; + let key = scan.string()?; + if !keys.insert(key.clone()) { + return scan.error(start, JsonErrorKind::DuplicateKey { key }); + } + scan.skip_white_space(); + if scan.peek() != Some(b':') { + return scan.unexpected("':'"); + } + scan.pos += 1; + Ok(key) +} diff --git a/crates/lumen/src/json/scan.rs b/crates/lumen/src/json/scan.rs new file mode 100644 index 0000000..807d4f8 --- /dev/null +++ b/crates/lumen/src/json/scan.rs @@ -0,0 +1,188 @@ +//! The reader's lexical level: a cursor over the bytes that reads one string, number or word. + +use super::error::{JsonError, JsonErrorKind}; + +/// A cursor over the input. +pub(super) struct Scanner<'a> { + pub(super) bytes: &'a [u8], + pub(super) pos: usize, +} + +type Scan = Result; + +impl<'a> Scanner<'a> { + pub(super) const fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + pub(super) fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + pub(super) fn skip_white_space(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.pos += 1; + } + } + + pub(super) fn error(&self, offset: usize, kind: JsonErrorKind) -> Scan { + Err(JsonError::at(self.bytes, offset, kind)) + } + + /// The error for what is at the cursor: the end of the input, or a byte that cannot come. + pub(super) fn unexpected(&self, expected: &'static str) -> Scan { + match self.peek() { + None => self.error(self.pos, JsonErrorKind::UnexpectedEnd { expected }), + Some(byte) => self.error(self.pos, JsonErrorKind::UnexpectedByte { byte, expected }), + } + } + + /// The word `true`, `false` or `null`, whose first byte is at the cursor. + pub(super) fn word(&mut self, word: &'static [u8]) -> Scan<()> { + for &expected in word { + if self.peek() != Some(expected) { + return self.unexpected("the rest of a word such as true, false or null"); + } + self.pos += 1; + } + Ok(()) + } + + fn digits(&mut self) -> Scan<()> { + if !matches!(self.peek(), Some(b'0'..=b'9')) { + return self.unexpected("a digit"); + } + while matches!(self.peek(), Some(b'0'..=b'9')) { + self.pos += 1; + } + Ok(()) + } + + /// A number token, validated against the JSON grammar and returned as written. + pub(super) fn number(&mut self) -> Scan { + let start = self.pos; + if self.peek() == Some(b'-') { + self.pos += 1; + } + if self.peek() == Some(b'0') { + self.pos += 1; + } else { + self.digits()?; + } + if self.peek() == Some(b'.') { + self.pos += 1; + self.digits()?; + } + if matches!(self.peek(), Some(b'e' | b'E')) { + self.pos += 1; + if matches!(self.peek(), Some(b'+' | b'-')) { + self.pos += 1; + } + self.digits()?; + } + let token = self.bytes.get(start..self.pos).unwrap_or_default(); + // The token is ASCII by construction; the check only keeps this function total. + match core::str::from_utf8(token) { + Ok(text) => Ok(text.to_owned()), + Err(_) => self.error(start, JsonErrorKind::InvalidUtf8), + } + } + + /// Four hexadecimal digits after `\u`, as a code unit. + fn hex4(&mut self) -> Scan { + let mut unit = 0u16; + for _ in 0..4 { + let digit = match self.peek() { + Some(b @ b'0'..=b'9') => b - b'0', + Some(b @ b'a'..=b'f') => b - b'a' + 10, + Some(b @ b'A'..=b'F') => b - b'A' + 10, + _ => return self.unexpected("a hexadecimal digit"), + }; + unit = unit * 16 + u16::from(digit); + self.pos += 1; + } + Ok(unit) + } + + /// The character of a `\u` escape whose backslash is at `backslash` and whose four digits + /// were read: the character itself, or a pair's second escape too for a high surrogate. + fn unicode_escape(&mut self, backslash: usize) -> Scan { + let unit = self.hex4()?; + let lone = JsonErrorKind::LoneSurrogate { unit }; + let combined = match unit { + 0xD800..=0xDBFF => { + if self.peek().is_none() { + return self.unexpected("the low half of a surrogate pair"); + } + let second = self.pos; + if self.bytes.get(second..second + 2) != Some(b"\\u") { + return self.error(backslash, lone); + } + self.pos += 2; + let low = self.hex4()?; + if !(0xDC00..=0xDFFF).contains(&low) { + return self.error(backslash, lone); + } + 0x1_0000 + ((u32::from(unit) - 0xD800) << 10) + (u32::from(low) - 0xDC00) + } + 0xDC00..=0xDFFF => return self.error(backslash, lone), + _ => u32::from(unit), + }; + match char::from_u32(combined) { + Some(c) => Ok(c), + None => self.error(backslash, lone), + } + } + + /// A string whose opening quote is at the cursor, with its escapes decoded. + pub(super) fn string(&mut self) -> Scan { + self.pos += 1; + let mut out = String::new(); + loop { + let run = self.pos; + while let Some(b) = self.peek() { + if b == b'"' || b == b'\\' || b < 0x20 { + break; + } + self.pos += 1; + } + let raw = self.bytes.get(run..self.pos).unwrap_or_default(); + match core::str::from_utf8(raw) { + Ok(text) => out.push_str(text), + Err(e) => return self.error(run + e.valid_up_to(), JsonErrorKind::InvalidUtf8), + } + match self.peek() { + None => return self.unexpected("the closing quote of a string"), + Some(b'"') => { + self.pos += 1; + return Ok(out); + } + Some(b'\\') => { + let backslash = self.pos; + self.pos += 1; + let c = match self.peek() { + Some(b'"') => '"', + Some(b'\\') => '\\', + Some(b'/') => '/', + Some(b'b') => '\u{8}', + Some(b'f') => '\u{c}', + Some(b'n') => '\n', + Some(b'r') => '\r', + Some(b't') => '\t', + Some(b'u') => { + self.pos += 1; + out.push(self.unicode_escape(backslash)?); + continue; + } + _ => return self.unexpected("an escape: one of \" \\ / b f n r t u"), + }; + self.pos += 1; + out.push(c); + } + Some(byte) => { + return self.error(self.pos, JsonErrorKind::ControlCharacter { byte }); + } + } + } + } +} diff --git a/crates/lumen/src/json/tests.rs b/crates/lumen/src/json/tests.rs new file mode 100644 index 0000000..378f490 --- /dev/null +++ b/crates/lumen/src/json/tests.rs @@ -0,0 +1,184 @@ +//! Unit tests of the reader: limits exactly on and one past, every truncation, positions. +//! +//! Oracles: the limits and positions are written by hand from the contract in the module +//! documentation, never computed by the reader. Truncation uses documents whose validity is +//! known by construction: a proper prefix of a document whose top level is a container or a +//! string is never valid JSON, since the closing bracket or quote is the last byte. + +use super::*; + +fn parse(text: &[u8]) -> Result { + parse_json(text, JsonLimits::default()) +} + +fn nested(depth: usize) -> Vec { + let mut doc = vec![b'['; depth]; + doc.extend(vec![b']'; depth]); + doc +} + +#[test] +fn the_reader_has_an_identity() { + assert_eq!(JSON_READER_ID, "lumen-json-reader/1"); +} + +#[test] +fn nesting_exactly_at_the_depth_limit_is_read() { + let limits = JsonLimits::new(5, 1000).unwrap(); + assert!(parse_json(&nested(5), limits).is_ok()); +} + +#[test] +fn nesting_one_past_the_depth_limit_is_refused_at_the_bracket() { + let limits = JsonLimits::new(5, 1000).unwrap(); + let error = parse_json(&nested(6), limits).unwrap_err(); + assert_eq!(error.kind(), &JsonErrorKind::TooDeep { limit: 5 }); + assert_eq!((error.offset(), error.line(), error.column()), (5, 1, 6)); +} + +#[test] +fn objects_count_toward_depth_like_arrays() { + let limits = JsonLimits::new(2, 1000).unwrap(); + assert!(parse_json(br#"{"a":{}}"#, limits).is_ok()); + let error = parse_json(br#"{"a":{"b":{}}}"#, limits).unwrap_err(); + assert_eq!(error.kind(), &JsonErrorKind::TooDeep { limit: 2 }); + assert_eq!(error.offset(), 11); +} + +#[test] +fn depth_zero_admits_scalars_only() { + let limits = JsonLimits::new(0, 1000).unwrap(); + assert!(parse_json(b"1", limits).is_ok()); + assert_eq!(parse_json(b"[]", limits).unwrap_err().offset(), 0); +} + +#[test] +fn too_deep_is_refused_before_the_rest_is_read() { + // Unterminated and past the limit: the depth error comes first, so nothing deeper was stored. + let limits = JsonLimits::new(3, 1000).unwrap(); + let error = parse_json(b"[[[[[[[[", limits).unwrap_err(); + assert_eq!(error.kind(), &JsonErrorKind::TooDeep { limit: 3 }); +} + +#[test] +fn the_depth_ceiling_is_accepted_and_one_past_it_is_refused() { + assert!(JsonLimits::new(JSON_MAX_DEPTH_CEILING, 1).is_ok()); + let error = JsonLimits::new(JSON_MAX_DEPTH_CEILING + 1, 1).unwrap_err(); + assert_eq!(error.max_depth(), 1025); +} + +#[test] +fn a_document_at_the_ceiling_is_read_and_dropped_on_a_test_thread() { + let limits = JsonLimits::new(JSON_MAX_DEPTH_CEILING, 1 << 20).unwrap(); + let value = parse_json(&nested(JSON_MAX_DEPTH_CEILING), limits).unwrap(); + assert_eq!(value.clone(), value); +} + +#[test] +fn input_exactly_at_the_size_limit_is_read() { + let limits = JsonLimits::new(4, 7).unwrap(); + assert!(parse_json(b"[1,2,3]", limits).is_ok()); +} + +#[test] +fn input_one_byte_past_the_size_limit_is_refused_before_being_read() { + let limits = JsonLimits::new(4, 7).unwrap(); + // Invalid after byte 7 as well, but the size is what is reported, at the first byte past it. + let error = parse_json(b"[1,2,3]]", limits).unwrap_err(); + assert_eq!(error.kind(), &JsonErrorKind::TooLarge { length: 8, limit: 7 }); + assert_eq!((error.offset(), error.line(), error.column()), (7, 1, 8)); +} + +#[test] +fn a_zero_byte_limit_still_reports_empty_input_as_an_end() { + let limits = JsonLimits::new(4, 0).unwrap(); + let error = parse_json(b"", limits).unwrap_err(); + assert!(matches!(error.kind(), JsonErrorKind::UnexpectedEnd { .. })); + assert!(matches!( + parse_json(b"1", limits).unwrap_err().kind(), + JsonErrorKind::TooLarge { .. } + )); +} + +#[test] +fn the_defaults_are_the_stated_values() { + let d = JsonLimits::default(); + assert_eq!((d.max_depth(), d.max_bytes()), (128, 16_777_216)); +} + +const DOCUMENTS: &[&str] = &[ + r#"{"a":[1,-2.5e+3,true,false,null,"xĂ©đŸ˜€\n"],"b":{"c":{}},"d":[]}"#, + "[\n {\"k\": \"v\"},\n [[], [[]]],\n 0\n]", + r#""a string with \"escapes\" and \\ and Ă© and 😀""#, + r#"{"Ă©đŸ˜€":"\/","":[0,-0,0.0,1E400]}"#, +]; + +#[test] +fn every_proper_prefix_of_a_container_or_string_document_is_refused_with_a_position() { + for doc in DOCUMENTS { + let bytes = doc.as_bytes(); + assert!(parse(bytes).is_ok(), "{doc}"); + for cut in 0..bytes.len() { + let error = parse(&bytes[..cut]).expect_err(&format!("{doc} cut at {cut}")); + assert!(error.line() >= 1 && error.column() >= 1); + assert!(error.offset() <= cut, "{doc} cut at {cut}: {error}"); + } + } +} + +#[test] +fn every_truncation_of_a_scalar_document_is_refused_or_is_itself_valid() { + // Oracle: the valid prefixes of `-12.5e+3` are those that end in a digit and are complete + // numbers: `-1`, `-12`, `-12.5` and `-12.5e+3`; the others must be refused. + let doc = b"-12.5e+3"; + let valid = [2usize, 3, 5, 8]; + for cut in 1..=doc.len() { + assert_eq!(parse(&doc[..cut]).is_ok(), valid.contains(&cut), "cut at {cut}"); + } +} + +#[test] +fn positions_count_characters_in_columns_and_line_feeds_in_lines() { + // Line 2 is `Ă©đŸ˜€ x`: the stray `x` follows 3 characters (5 bytes... Ă© is 2, 😀 is 4). + let error = parse("[\"\",\n\"Ă©đŸ˜€\" x]".as_bytes()).unwrap_err(); + assert_eq!((error.line(), error.column()), (2, 5)); + assert_eq!(error.offset(), 5 + 1 + 2 + 4 + 1 + 1 - 1 + 0); +} + +#[test] +fn a_carriage_return_is_a_character_of_its_line() { + let error = parse(b"[1,\r\n2 3]").unwrap_err(); + assert_eq!((error.line(), error.column()), (2, 3)); + let error = parse(b"[1,\r2 3]").unwrap_err(); + assert_eq!((error.line(), error.column()), (1, 6)); +} + +#[test] +fn an_error_at_the_end_is_at_the_length() { + let error = parse(b"[1,\n").unwrap_err(); + assert_eq!((error.offset(), error.line(), error.column()), (4, 2, 1)); +} + +#[test] +fn display_names_line_and_column() { + let text = parse(b"[1,]").unwrap_err().to_string(); + assert!(text.starts_with("line 1, column 4: "), "{text}"); +} + +#[test] +fn accessors_read_the_tree() { + let v = parse(br#"{"a":[1],"s":"t","n":2.50}"#).unwrap(); + assert_eq!(v.get("a").and_then(JsonValue::as_array).map(<[_]>::len), Some(1)); + assert_eq!(v.get("s").and_then(JsonValue::as_str), Some("t")); + assert_eq!(v.get("n").and_then(JsonValue::as_number), Some("2.50")); + assert_eq!(v.as_object().map(<[_]>::len), Some(3)); + assert_eq!(v.get("zz"), None); + assert_eq!(JsonValue::Null.get("a"), None); +} + +#[test] +fn members_keep_document_order() { + let v = parse(br#"{"b":1,"a":2,"c":3}"#).unwrap(); + let keys: Vec<&str> = v.as_object().unwrap().iter().map(|(k, _)| k.as_str()).collect(); + assert_eq!(keys, ["b", "a", "c"]); +} diff --git a/crates/lumen/src/json/value.rs b/crates/lumen/src/json/value.rs new file mode 100644 index 0000000..2e03643 --- /dev/null +++ b/crates/lumen/src/json/value.rs @@ -0,0 +1,69 @@ +//! The tree the reader returns: [`JsonValue`]. + +/// A JSON value. A number is its source text, validated and unconverted. +/// +/// An object keeps its members in document order, with no key twice (the reader refuses a +/// duplicate). Equality is structural, so `1.0` and `1` are different numbers here: the reader +/// has no opinion on what a token means. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JsonValue { + /// `null`. + Null, + /// `true` or `false`. + Bool(bool), + /// A number token exactly as written, for example `-0`, `1.50` or `1E400`. + Number(String), + /// A string with its escapes decoded. + String(String), + /// An array's elements in order. + Array(Vec), + /// An object's members in document order. + Object(Vec<(String, JsonValue)>), +} + +impl JsonValue { + /// The member of an object named `key`, or `None` for another kind of value or a missing key. + #[must_use] + pub fn get(&self, key: &str) -> Option<&Self> { + match self { + Self::Object(members) => members.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + /// The text of a string, or `None` for another kind of value. + #[must_use] + pub fn as_str(&self) -> Option<&str> { + match self { + Self::String(text) => Some(text), + _ => None, + } + } + + /// The source text of a number, or `None` for another kind of value. + #[must_use] + pub fn as_number(&self) -> Option<&str> { + match self { + Self::Number(text) => Some(text), + _ => None, + } + } + + /// The elements of an array, or `None` for another kind of value. + #[must_use] + pub fn as_array(&self) -> Option<&[Self]> { + match self { + Self::Array(items) => Some(items), + _ => None, + } + } + + /// The members of an object in document order, or `None` for another kind of value. + #[must_use] + pub fn as_object(&self) -> Option<&[(String, Self)]> { + match self { + Self::Object(members) => Some(members), + _ => None, + } + } +} diff --git a/crates/lumen/src/lib.rs b/crates/lumen/src/lib.rs index 4bc96b9..496a3c9 100644 --- a/crates/lumen/src/lib.rs +++ b/crates/lumen/src/lib.rs @@ -50,6 +50,7 @@ mod builtin_scene; #[path = "../tests/generator/mod.rs"] mod generator; mod image; +mod json; pub mod mesh; mod png; #[cfg(test)] @@ -73,6 +74,10 @@ pub const BUNNY_REV: &str = "2cd551e248d55407a8f27e84dcb26d1b1d8768bd"; pub use builtin_scene::{BuiltinScene, BuiltinSceneError, ParseSceneError}; pub use image::{ImageSize, LengthError, Rgba8Image, SizeError, MAX_DIMENSION, MAX_PIXELS}; +pub use json::{ + parse_json, JsonError, JsonErrorKind, JsonLimitError, JsonLimits, JsonValue, + JSON_DEFAULT_MAX_BYTES, JSON_DEFAULT_MAX_DEPTH, JSON_MAX_DEPTH_CEILING, JSON_READER_ID, +}; pub use png::{png_len, write_png, PngError}; pub use provenance::{ProvenanceValue, RenderProvenance, PROVENANCE_FIELDS}; pub use reference::{reference_scene, REFERENCE_SCENE_ID};