From d830aeab2b2521b21b36541f0c9657c525cf9448 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 19:41:57 +1100 Subject: [PATCH 01/14] feat(stack-encrypt): a data plan field says how it stores a null A plan field may now carry "null": "sealed" or "sql" (#1298), so an optional value can be encrypted without a sentinel. - "sealed" seals a null like a value, under the field's label, as vitaminc 0.8's typed null of the field's type, so its ciphertext is as long as a present value's and the stored record does not show which rows are absent. It opens as a null. The field must declare a type of fixed width (a bool's plain null is already as long as its value) and derive no term, since a null has none yet. - "sql" stores nothing for a null, or for a row with no entry for the field: the stored record has no entry, so the host writes SQL NULL. A stored record with no entry for the field opens it as a null. Rows of a batch are run by one lowered plan per set of fields they leave out, and the groups' requests are merged into the one call. A field with no "null" behaves as before: every row must have it, and a typed field refuses a null. Refs #1298. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/kind.rs | 36 +- packages/stack-encrypt/src/dynamic/mod.rs | 19 +- packages/stack-encrypt/src/dynamic/record.rs | 770 +++++++++++++++--- .../stack-encrypt/src/dynamic/value_frame.rs | 27 + 4 files changed, 754 insertions(+), 98 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/kind.rs b/packages/stack-encrypt/src/dynamic/kind.rs index 3606b9c3a..a36fe43ef 100644 --- a/packages/stack-encrypt/src/dynamic/kind.rs +++ b/packages/stack-encrypt/src/dynamic/kind.rs @@ -38,7 +38,7 @@ use std::fmt; use std::str::FromStr; -use vitaminc_aead::plaintext::{Float, Int, Logical, Physical, Raw, TimeUnit}; +use vitaminc_aead::plaintext::{FixedType, Float, Int, Logical, Physical, Raw, TimeUnit, Width}; use vitaminc_aead::Encrypt; use super::term::OneScalar; @@ -461,6 +461,40 @@ impl ValueKind { ValueKind::Object => "object", } } + + /// How an absent value of this kind is sealed so that it is as long as + /// a present one (a plan's `"null": "sealed"`, #1298): `Some(Some(ty))` + /// for a kind of fixed width, sealed as a typed null of `ty` (vitaminc + /// 0.8); `Some(None)` for a boolean, whose leaf is one byte, as the + /// plain null is; `None` for a kind of no fixed width (text, bytes, a + /// container), whose absence no padding can hide. + pub(crate) const fn sealed_null(self) -> Option> { + let plain = FixedType::plain; + Some(match self { + ValueKind::Bool => None, + ValueKind::Int8 => plain(Width::I8), + ValueKind::Int16 => plain(Width::I16), + ValueKind::Int32 => plain(Width::I32), + ValueKind::Int64 => plain(Width::I64), + ValueKind::Int128 => plain(Width::I128), + ValueKind::UInt8 => plain(Width::U8), + ValueKind::UInt16 => plain(Width::U16), + ValueKind::UInt32 => plain(Width::U32), + ValueKind::UInt64 => plain(Width::U64), + ValueKind::UInt128 => plain(Width::U128), + ValueKind::Float32 => plain(Width::F32), + ValueKind::Float64 => plain(Width::F64), + // The widths the value model holds each logical type at. + ValueKind::Decimal => FixedType::new(Some(Logical::Decimal { scale: 0 }), Width::I128), + ValueKind::Date => FixedType::new(Some(Logical::Date), Width::I32), + ValueKind::Timestamp => { + FixedType::new(Some(Logical::Timestamp(TimeUnit::Nanos)), Width::I128) + } + ValueKind::String | ValueKind::Bytes | ValueKind::Array | ValueKind::Object => { + return None + } + }) + } } impl fmt::Display for ValueKind { diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 1cdd8164d..a7a72ac2c 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -106,7 +106,7 @@ pub use query::{ check_query, field_operators, query_term, target_limit, verify, Fidelity, OpSpec, Operand, OperatorEntry, QueryTerm, }; -pub use record::{FieldPlan, Output, Plan}; +pub use record::{FieldPlan, NullMode, Output, Plan}; pub use target::{NoTargets, TargetDescriptor, TargetError, TargetResolver}; pub use term::{term, TermBytes}; pub use value_frame::{Children, FromFrameError, Modeled, Opening, ValueFrame}; @@ -512,8 +512,8 @@ reasons! { /// A field is named twice in the plan. DuplicateField => "duplicate_field", "a field is named twice"; /// A field spec has a key other than `"context"`, `"outputs"`, - /// `"target"` and `"type"`. - UnknownKey => "unknown_key", r#"a key other than "context", "outputs", "target" and "type""#; + /// `"target"`, `"type"` and `"null"`. + UnknownKey => "unknown_key", r#"a key other than "context", "outputs", "target", "type" and "null""#; /// A key is given twice: in a field spec, or in a map inside a source /// value or a stored record. RepeatedKey => "repeated_key", "a key is given twice"; @@ -602,6 +602,19 @@ reasons! { /// Statistics are not a row count with an optional histogram and /// confidence ([`Stats::from_frame`](crate::op::Stats::from_frame)). NotStats => "not_stats", "the statistics are not a row count with an optional histogram and confidence"; + /// `"null"` is not `"sealed"` or `"sql"`. + UnknownNull => "unknown_null", r#""null" is not "sealed" or "sql""#; + /// A passthrough field says how to store a null: it carries one as + /// it is. + NullPassthrough => "null_passthrough", r#"a passthrough carries a null as it is, and takes no "null""#; + /// A field with `"null": "sealed"` derives a term, or names an EQL + /// type that carries one: a null has no term yet (#1298), so its + /// absence would show in the term column. + SealedNullIndex => "sealed_null_index", r#"a sealed null has no term yet, so a field with one takes "null": "sql""#; + /// A field with `"null": "sealed"` declares no type, or one of no + /// fixed width (text, bytes, a container), whose absence the length + /// of its ciphertext would show. + SealedNullType => "sealed_null_type", "a sealed null needs a declared type of fixed width: a bool, a number, a date, a timestamp or a decimal"; } } diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 576c9b806..c3e05e6e3 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -140,6 +140,7 @@ //! that a round-trip invariant rather than data loss. use super::value_frame::{split_rows, FieldFault, SourceRow, SourceRows}; +use std::collections::BTreeSet; use std::fmt; use serde::de::{Deserializer, MapAccess, Visitor}; @@ -156,7 +157,7 @@ use super::{ not_admitted, Error, NoTargets, Reason, Scope, TargetError, TargetResolver, TermBytes, }; use crate::plan::{FieldValues, FieldsBuilder, Opens, Runs}; -use crate::target::{CallerContext, DeclaredContext, Decryption, Encrypted, IndexSpec}; +use crate::target::{CallerContext, DeclaredContext, Encrypted, IndexSpec}; use crate::{ CipherText, ContextPiece, KeysetCipher, Label, NonEmpty, PassthroughFrame, Pending, StackCipherText, @@ -218,6 +219,38 @@ enum Verb { Target, } +/// How a field stores a value that is not there: what a plan's `"null"` +/// says (#1298). A field with no `"null"` needs every row to have it, and +/// refuses a null where it declares a type, as before. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum NullMode { + /// `"sealed"`: a null is sealed like a value, under the field's label, + /// as a typed null as long as a value of the field's type (vitaminc + /// 0.8), so the stored record does not say which rows are absent. It + /// opens as a null. Only for a field that seals and derives no term (a + /// null has none yet), of a declared type of fixed width: a boolean, a + /// number, a date, a timestamp or a decimal. A target field takes it + /// when its EQL type carries no term. + Sealed, + /// `"sql"`: a null, or a row with no entry for the field, produces + /// nothing: the stored record has no entry for the field, so the host + /// stores SQL `NULL` in its columns, which shows that the value is + /// absent. A stored record with no entry for the field opens it as a + /// null. Any field but a passthrough takes it. + Sql, +} + +impl NullMode { + /// How a plan spells the mode: `"sealed"` or `"sql"`. Frozen wire + /// format. + pub const fn name(self) -> &'static str { + match self { + NullMode::Sealed => "sealed", + NullMode::Sql => "sql", + } + } +} + /// One field of a record plan: what to call it, what label to seal it /// under, what to produce for it, and, optionally, what type its values are. /// @@ -239,6 +272,7 @@ pub struct FieldPlan { outputs: Vec, target: Option, field_type: Option, + null: Option, } impl FieldPlan { @@ -294,6 +328,7 @@ impl FieldPlan { outputs, target: None, field_type: None, + null: None, }) } @@ -329,6 +364,7 @@ impl FieldPlan { outputs: Vec::new(), target: Some(target), field_type: None, + null: None, }) } @@ -357,6 +393,19 @@ impl FieldPlan { Ok(self) } + /// Say how the field stores a value that is not there ([`NullMode`]). + /// Whether the field can is decided when the plan is built + /// ([`Plan::new_with`]), once its type is known. + pub fn with_null(mut self, null: NullMode) -> Self { + self.null = Some(null); + self + } + + /// How the field stores a value that is not there, if the plan says. + pub fn null(&self) -> Option { + self.null + } + /// The field's name — its key in the source and in the result. pub fn name(&self) -> &str { &self.name @@ -460,6 +509,28 @@ impl FieldPlan { verb: self.verb(), keys: self.indexes().iter().map(IndexSpec::key).collect(), kind: self.field_type, + null: self.null, + } + } + + /// Why the field cannot store a null as its [`NullMode`] says, if it + /// cannot: a passthrough carries a null as it is; a sealed null needs + /// a type of fixed width and no term. A target field's terms are its + /// type's, which [`Plan::new_with`] checks. + fn null_refusal(&self) -> Option { + let null = self.null?; + if self.verb() == Verb::Passthrough { + return Some(Reason::NullPassthrough); + } + if null == NullMode::Sql { + return None; + } + if !matches!(self.verb(), Verb::Encrypt | Verb::Target) { + return Some(Reason::SealedNullIndex); + } + match self.field_type.and_then(ValueKind::sealed_null) { + Some(_) => None, + None => Some(Reason::SealedNullType), } } @@ -668,6 +739,13 @@ impl Plan { (None, Some(kind)) => field.field_type = Some(kind), _ => {} } + // A sealed null has no term, so its type must carry none. + if field.null == Some(NullMode::Sealed) && !descriptor.indexes.is_empty() { + return Err(Error::Plan { + field: Some(field.name.clone()), + reason: Reason::SealedNullIndex, + }); + } } Self::build(Context::Label(context), fields) } @@ -765,6 +843,9 @@ impl Plan { if field.extension != extension { return Err(refuse(Reason::MixedContexts)); } + if let Some(reason) = field.null_refusal() { + return Err(refuse(reason)); + } // An indexed field declares its type: every value's term derives // from the one declared kind, never from whatever tag each value // arrived with. A target field's kind is the type's own. @@ -848,6 +929,16 @@ impl Plan { /// `context_field`, which carries it as a passthrough and reads it as /// the context; the loop then skips it. fn lower(&self) -> Result, crate::Error> { + self.lower_without(&Absent::new()) + } + + /// The fields plan this declaration lowers to without the fields in + /// `absent`: what runs a row that leaves them out (a `"null": "sql"` + /// field that is null or not there). Their names are the plan's own. + fn lower_without( + &self, + absent: &Absent, + ) -> Result, crate::Error> { let mut builder = match &self.context { Context::Label(label) => crate::Plan::context(label.clone()).fields::(), Context::Field(name) => { @@ -857,7 +948,10 @@ impl Plan { for field in &self.fields { // The context field is declared by `context_field` above; a // target field is the resolver's. - if self.context_field() == Some(field.name.as_str()) || field.is_target() { + if self.context_field() == Some(field.name.as_str()) + || field.is_target() + || absent.contains(&field.name) + { continue; } builder = declare(builder, field); @@ -987,6 +1081,16 @@ pub fn plan(frame: &[u8]) -> Result { /// decides nothing about the bytes (see the /// [module docs](self#what-a-fields-type-decides-and-what-it-does-not)). /// +/// `"null"`, when given, is `"sealed"` or `"sql"`: how the field stores a +/// value that is not there ([`NullMode`], #1298). Without it a row must +/// have the field, and a typed field refuses a null. `"sealed"` seals a +/// null as a typed null as long as a value, so the stored record does not +/// show it; the field must declare a type of fixed width (a bool, a number, +/// a date, a timestamp or a decimal), have no term output, and, as a target +/// field, name an EQL type with no term. `"sql"` stores nothing for a null +/// or a missing field, so the host's columns hold SQL `NULL`; any field but +/// a passthrough takes it. +/// /// `` is read by [`super::context`](super::context()) and must be /// the field's label, optionally extended; the /// [module docs](self#what-a-fields-context-must-be) give the shape. @@ -1023,7 +1127,9 @@ pub fn plan(frame: &[u8]) -> Result { /// [`Error::Plan`] for a plan that is not an object of field specs /// (including bytes that are not one frame, which is how a field or key /// given twice is refused: the frame refuses it), an empty plan, a spec with -/// a key other than `"context"`, `"outputs"`, `"target"` and `"type"`, +/// a key other than `"context"`, `"outputs"`, `"target"`, `"type"` and +/// `"null"`, a `"null"` other than `"sealed"` or `"sql"`, a field that cannot +/// store a null as its `"null"` says ([`NullMode`]), /// missing `"context"`, having neither `"outputs"` nor `"target"` or having /// both, an output list that is not a list of outputs (above), is empty, /// names an output key twice or names `"passthrough"` beside another @@ -1052,9 +1158,13 @@ pub fn plan_with(frame: &[u8], resolver: &(impl TargetResolver + ?Sized)) -> Res What::Outputs(outputs) => FieldPlan::new(name, context, outputs)?, What::Target(target) => FieldPlan::with_target(name, context, target)?, }; - fields.push(match spec.field_type { + let field = match spec.field_type { Some(KindName(field_type)) => field.with_type(field_type)?, None => field, + }; + fields.push(match spec.null { + Some(NullName(null)) => field.with_null(null), + None => field, }); } // The whole-plan rules are the constructors', so a parsed plan and a @@ -1132,7 +1242,7 @@ fn entry_key(frame: &[u8], position: usize) -> Option { } /// The keys a field's spec reads ([`FieldSpecWire`]). -const SPEC_KEYS: [&str; 4] = ["context", "outputs", "target", "type"]; +const SPEC_KEYS: [&str; 5] = ["context", "outputs", "target", "type", "null"]; /// The keys of the spec `field` holds in the plan `frame`, or `None` when /// that spec is not an object (or the plan is not one frame of unique keys). @@ -1163,6 +1273,7 @@ struct FieldSpec { context: super::context::Context, what: What, field_type: Option, + null: Option, } enum What { @@ -1180,6 +1291,8 @@ struct FieldSpecWire { target: Option, #[serde(default, deserialize_with = "present", rename = "type")] field_type: Option, + #[serde(default, deserialize_with = "present")] + null: Option, } impl TryFrom for FieldSpec { @@ -1196,6 +1309,7 @@ impl TryFrom for FieldSpec { context: wire.context, what, field_type: wire.field_type, + null: wire.null, }) } } @@ -1250,6 +1364,23 @@ impl TryFrom<&str> for KindName { } } +/// A `"null"`: the name of a [`NullMode`]. +#[derive(Deserialize)] +#[serde(try_from = "&str")] +struct NullName(NullMode); + +impl TryFrom<&str> for NullName { + type Error = Reason; + + fn try_from(name: &str) -> Result { + match name { + "sealed" => Ok(NullName(NullMode::Sealed)), + "sql" => Ok(NullName(NullMode::Sql)), + _ => Err(Reason::UnknownNull), + } + } +} + /// The parser's refusal of a plan the frame reader refused: the field from /// the error's path, and the reason from where in the field it happened, /// or from the reason a field's own rule gave. @@ -1279,6 +1410,7 @@ fn plan_refusal(error: &vitaminc_aead_frame::serde::Error, frame: &[u8]) -> Erro Some(Key(key)) if key == "outputs" && path.len() == 2 => Reason::OutputsNotList, Some(Key(key)) if key == "outputs" => Reason::UnknownOutput, Some(Key(key)) if key == "target" => Reason::InvalidTarget, + Some(Key(key)) if key == "null" => Reason::UnknownNull, // `"type"`: the one other key a spec reads, and so the only other // key an error inside a spec's value is placed under (an unknown // key is never on the path). @@ -1396,46 +1528,80 @@ pub fn encrypt_with<'a, K: 'static>( resolver: &(impl TargetResolver + ?Sized), ) -> Result, Error> { let rows = source_rows(source, plan)?; - let lowered = plan.lower::().map_err(|_| Error::Internal)?; let extend = plan.declared_context(); let shape = plan.shape(); + let lower = |absent: &Absent| plan.lower_without::(absent).map_err(|_| Error::Internal); // Every target field of every row, in row-major order, run by the // resolver; `all` merges their requests with the lowered plan's below. + // A row runs the target fields it does not leave out, so each row's + // count is its own. let mut pendings = Vec::new(); - let mut targets = |row: Row| -> Result { - for (field, value) in plan.target_fields().zip(row.targets) { + let mut targets = |row: Row| -> Result<(Absent, FieldValues, usize), Error> { + let fields = plan + .target_fields() + .filter(|field| !row.absent.contains(&field.name)); + let mut count = 0; + for (field, value) in fields.zip(row.targets) { let name = field.target().unwrap_or_default(); pendings.push( resolver .encrypt(name, cipher, field.label(), value) .map_err(|error| Error::Target(name_target(error, &field.name)))?, ); + count += 1; } - Ok(row.values) + Ok((row.absent, row.values, count)) }; Ok(match rows { Rows::One(row) => { - let values = targets(row)?; + let (absent, values, _) = targets(row)?; + let lowered = lower(&absent)?; let eql = Pending::all(cipher, pendings); Runs::::pending(&lowered, cipher, &values, None, extend) .zip(eql) - .try_map(move |(values, eql)| shape_record(values, eql, &shape)) + .try_map(move |(values, eql)| shape_record(values, eql, &shape, &absent)) } Rows::Batch(rows) => { - let values = rows + let rows = rows .into_iter() .map(&mut targets) - .collect::, Error>>()?; + .collect::, Error>>()?; + let len = rows.len(); + let mut per_row = Vec::with_capacity(len); + let groups = by_absent( + rows.into_iter() + .map(|(absent, values, count)| { + per_row.push((absent.clone(), count)); + (absent, values) + }) + .collect(), + ); + // One lowered plan per group of rows that leave out the same + // fields, every group's requests merged into the one call. + let mut runs = Vec::with_capacity(groups.len()); + let mut positions = Vec::with_capacity(groups.len()); + for (absent, at, values) in groups { + let lowered = lower(&absent)?; + runs.push(Runs::<[FieldValues], K>::pending( + &lowered, + cipher, + &values, + None, + extend.clone(), + )); + positions.push(at); + } let eql = Pending::all(cipher, pendings); - let per_row = plan.target_fields().count(); - Runs::<[FieldValues], K>::pending(&lowered, cipher, &values, None, extend) + Pending::all(cipher, runs) .zip(eql) - .try_map(move |(rows, eql)| { + .try_map(move |(groups, eql)| { + let rows = in_row_order(len, positions.into_iter().zip(groups))?; let mut eql = eql.into_iter(); rows.into_iter() - .map(|values| { - let own: Vec> = eql.by_ref().take(per_row).collect(); - shape_record(values, own, &shape) + .zip(per_row) + .map(|(values, (absent, count))| { + let own: Vec> = eql.by_ref().take(count).collect(); + shape_record(values, own, &shape, &absent) }) .collect::, _>>() .map(CipherText::Sequence) @@ -1444,6 +1610,47 @@ pub fn encrypt_with<'a, K: 'static>( }) } +/// The rows of a batch grouped by the fields they leave out, each group +/// with the positions of its rows in the batch, in the order each group +/// first appears: one lowered plan runs each group +/// ([`Plan::lower_without`]). A batch with no `"null": "sql"` field is one +/// group, as it was one plan before. +fn by_absent(rows: Vec<(Absent, R)>) -> Vec<(Absent, Vec, Vec)> { + let mut groups: Vec<(Absent, Vec, Vec)> = Vec::new(); + for (at, (absent, row)) in rows.into_iter().enumerate() { + match groups.iter_mut().find(|(seen, _, _)| *seen == absent) { + Some((_, positions, members)) => { + positions.push(at); + members.push(row); + } + None => groups.push((absent, vec![at], vec![row])), + } + } + groups +} + +/// Each group's results back in batch order, the inverse of [`by_absent`]. +/// A group that answers with another count than it was given is the engine +/// answering with another shape than it was asked. +fn in_row_order( + len: usize, + groups: impl IntoIterator, Vec)>, +) -> Result, crate::Error> { + let mut rows: Vec> = std::iter::repeat_with(|| None).take(len).collect(); + for (positions, results) in groups { + if positions.len() != results.len() { + return Err(crate::Error::ResponseShape); + } + for (at, result) in positions.into_iter().zip(results) { + let slot = rows.get_mut(at).ok_or(crate::Error::ResponseShape)?; + *slot = Some(result); + } + } + rows.into_iter() + .map(|row| row.ok_or(crate::Error::ResponseShape)) + .collect() +} + /// Derive the EQL query value of one target field for one plaintext: the /// operand that matches stored values of the field, as JSON bytes, through /// the type's own query plan. A query derives no data key, so the pending @@ -1599,85 +1806,110 @@ pub fn decrypt_with<'a, K: 'static>( resolver: &(impl TargetResolver + ?Sized), ) -> Result, Error> { let rows = record_rows(record, plan)?; - let lowered = plan.lower::().map_err(|_| Error::Internal)?; let extend = plan.declared_context(); let shape = plan.shape(); + let lower = |absent: &Absent| plan.lower_without::(absent).map_err(|_| Error::Internal); let (cipher, keyset) = match &scope { Scope::Client(cipher) => (*cipher, None), Scope::Keyset(keyset) => (keyset.cipher(), Some(keyset.keyset_id())), }; - // Confine a pending to the scope's keyset, as the lowered plan's is: - // a target value sealed under another keyset is refused before any key - // is retrieved. - let confine = move |pending: Pending<'a, ValueFrame, K>| match keyset { - Some(keyset) => pending.scoped_to(keyset), - None => pending, - }; let mut pendings = Vec::new(); - let mut targets = |row: StoredRow| -> Result { - for (field, stored) in plan.target_fields().zip(row.targets) { + let mut targets = |row: StoredRow| -> Result<(Absent, FieldValues, usize), Error> { + let fields = plan + .target_fields() + .filter(|field| !row.absent.contains(&field.name)); + let mut count = 0; + for (field, stored) in fields.zip(row.targets) { let name = field.target().unwrap_or_default(); + // Confined to the scope's keyset, as the lowered plan's leaves + // are: a target value sealed under another keyset is refused + // before any key is retrieved. pendings.push(confine( resolver .decrypt(name, cipher, field.label(), &stored) .map_err(|error| Error::Target(name_target(error, &field.name)))?, + keyset, )); + count += 1; } - Ok(row.values) + Ok((row.absent, row.values, count)) }; Ok(match rows { Rows::One(row) => { - let values = targets(row)?; + let (absent, values, _) = targets(row)?; + let lowered = lower(&absent)?; let eql = Pending::all(cipher, pendings); - run( - scope, - Opens::::decryption(&lowered, values, expected, extend), + confine( + cipher.run_decryption(Opens::::decryption( + &lowered, values, expected, extend, + )), + keyset, ) .zip(eql) .try_map(move |(values, eql)| { - write_record(Rows::One(open_record(values, eql, &shape)?)) + write_record(Rows::One(open_record(values, eql, &shape, &absent)?)) }) } Rows::Batch(rows) => { - let values = rows + let rows = rows .into_iter() .map(&mut targets) - .collect::, Error>>()?; - let eql = Pending::all(cipher, pendings); - let per_row = plan.target_fields().count(); - run( - scope, - Opens::, K>::decryption(&lowered, values, expected, extend), - ) - .zip(eql) - .try_map(move |(rows, eql)| { - let mut eql = eql.into_iter(); - let rows = rows - .into_iter() - .map(|values| { - let own: Vec = eql.by_ref().take(per_row).collect(); - open_record(values, own, &shape) + .collect::, Error>>()?; + let len = rows.len(); + let mut per_row = Vec::with_capacity(len); + let groups = by_absent( + rows.into_iter() + .map(|(absent, values, count)| { + per_row.push((absent.clone(), count)); + (absent, values) }) - .collect::, _>>()?; - write_record(Rows::Batch(rows)) - }) + .collect(), + ); + let mut opens = Vec::with_capacity(groups.len()); + let mut positions = Vec::with_capacity(groups.len()); + for (absent, at, values) in groups { + let lowered = lower(&absent)?; + opens.push(confine( + cipher.run_decryption(Opens::, K>::decryption( + &lowered, + values, + expected.clone(), + extend.clone(), + )), + keyset, + )); + positions.push(at); + } + let eql = Pending::all(cipher, pendings); + Pending::all(cipher, opens) + .zip(eql) + .try_map(move |(groups, eql)| { + let rows = in_row_order(len, positions.into_iter().zip(groups))?; + let mut eql = eql.into_iter(); + let rows = rows + .into_iter() + .zip(per_row) + .map(|(values, (absent, count))| { + let own: Vec = eql.by_ref().take(count).collect(); + open_record(values, own, &shape, &absent) + }) + .collect::, _>>()?; + write_record(Rows::Batch(rows)) + }) } }) } -/// Run an opener through the scope the caller chose: the client opens -/// leaves from any of its keysets, a keyset cipher refuses a foreign one -/// before any key is retrieved. -fn run<'a, K: 'static, T: 'static>( - scope: Scope<'a, K>, - decryption: Decryption, +/// A pending confined to `keyset` when the caller's scope is one keyset: +/// the client opens leaves from any of its keysets, a keyset cipher +/// refuses a foreign one before any key is retrieved. +fn confine<'a, T: 'a, K>( + pending: Pending<'a, T, K>, + keyset: Option, ) -> Pending<'a, T, K> { - match scope { - Scope::Client(cipher) => cipher.run_decryption(decryption), - Scope::Keyset(keyset) => keyset - .cipher() - .run_decryption(decryption) - .scoped_to(keyset.keyset_id()), + match keyset { + Some(keyset) => pending.scoped_to(keyset), + None => pending, } } @@ -1695,8 +1927,10 @@ fn run<'a, K: 'static, T: 'static>( /// As [`encrypt`], minus the cipher. pub fn check_source(source: impl Encrypt, plan: &Plan) -> Result<(), Error> { let rows = source_rows(source, plan)?; - let lowered = plan.lower::<()>().map_err(|_| Error::Internal)?; let check = |row: &Row| { + let lowered = plan + .lower_without::<()>(&row.absent) + .map_err(|_| Error::Internal)?; Runs::::check(&lowered, &row.values, None).map_err(|error| { let (field, reason) = refusal(error); Error::Source { field, reason } @@ -1732,8 +1966,10 @@ pub fn check_record( expected: Option<&Label>, ) -> Result<(), Error> { let rows = record_rows(record, plan)?; - let lowered = plan.lower::<()>().map_err(|_| Error::Internal)?; let check = |row: &StoredRow| { + let lowered = plan + .lower_without::<()>(&row.absent) + .map_err(|_| Error::Internal)?; Opens::::check(&lowered, &row.values, expected).map_err( |error| match error { mismatch @ crate::Error::ContextMismatch { .. } => Error::Cipher(mismatch), @@ -1811,20 +2047,30 @@ enum Rows { Batch(Vec), } -/// One source row: the lowered plan's values, and each target field's -/// value in plan order, for the resolver. +/// One source row: the lowered plan's values, each target field's value in +/// plan order, for the resolver, and the fields the row leaves out. struct Row { values: FieldValues, targets: Vec, + absent: Absent, } -/// One stored row: the lowered plan's ciphertexts, and each target field's -/// EQL bytes in plan order, for the resolver. +/// One stored row: the lowered plan's ciphertexts, each target field's EQL +/// bytes in plan order, for the resolver, and the fields the row leaves +/// out. struct StoredRow { values: FieldValues, targets: Vec>, + absent: Absent, } +/// The `"null": "sql"` fields a row leaves out: null in a source, or with +/// no entry in a source or a stored record. Nothing is sealed, derived or +/// opened for them, so a row is run by the plan lowered without them +/// ([`Plan::lower_without`]), with the other rows that leave out the same +/// fields. +type Absent = BTreeSet; + /// Take the one entry named `name` out of a row, whatever order the row had /// it in. `None` if the row has no such entry — or has it twice: the maps /// this is used on (a row, a field's output map) are stripped here and never @@ -1948,19 +2194,41 @@ fn source_row(mut row: Vec<(String, ValueFrame)>, plan: &Plan) -> Result { + let _ = absent.insert(field.name.clone()); + continue; + } + // A null the field seals: as a typed null of its type, as long + // as a value of it, or as the plain null for a boolean, which + // is already as long as one. `Plan::build` refuses a sealed + // null on any other type, and on a field with a term. + Some(NullMode::Sealed) if value.is_null() => { + match field.field_type.and_then(ValueKind::sealed_null) { + Some(Some(ty)) => ValueFrame::typed_null(ty), + _ => value, + } + } + _ => { + check_field(&value, field)?; + value + } }; - check_field(&value, field)?; if field.is_target() { targets.push(value); } else { @@ -1972,7 +2240,11 @@ fn source_row(mut row: Vec<(String, ValueFrame)>, plan: &Plan) -> Result, kind: Option, + null: Option, } /// A slot of the record the engine built from the plan, at the type the @@ -2080,10 +2353,13 @@ fn shape_record( mut values: FieldValues, eql: Vec>, shape: &[FieldShape], + absent: &Absent, ) -> Result { let mut fields = Vec::with_capacity(shape.len()); let mut eql = eql.into_iter(); - for field in shape { + // A field the row leaves out has no entry: nothing was sealed or + // derived for it, and the host stores SQL `NULL`. + for field in shape.iter().filter(|field| !absent.contains(&field.name)) { let outputs = match field.verb { Verb::Target => { // The resolver answered one value per target field, in plan @@ -2143,16 +2419,25 @@ fn open_record( mut values: FieldValues, eql: Vec, shape: &[FieldShape], + absent: &Absent, ) -> Result, crate::Error> { let mut fields = Vec::with_capacity(shape.len()); let mut eql = eql.into_iter(); for field in shape.iter().filter(|field| field.verb != Verb::Index) { + // A field the stored row leaves out was null: it opens as one. + if absent.contains(&field.name) { + fields.push((field.name.clone(), ValueFrame::null())); + continue; + } let value = if field.verb == Verb::Target { eql.next().ok_or(crate::Error::ResponseShape)? } else { take_leaf(&mut values, &field.name)? }; - if let Some(kind) = field.kind { + // A field that stores a null opens one, sealed: a typed null + // opens as a plain null, as every frame carries one. + let null = field.null.is_some() && value.is_null(); + if let Some(kind) = field.kind.filter(|_| !null) { if value.kind() != Some(kind) { return Err(crate::PlanError::FieldType { field: field.name.clone(), @@ -2233,15 +2518,23 @@ fn record_rows(tree: StackCipherText, plan: &Plan) -> Result, Er fn record_row(mut row: Vec<(String, StackCipherText)>, plan: &Plan) -> Result { let mut values = FieldValues::new(); let mut targets = Vec::new(); + let mut absent = Absent::new(); for field in plan.fields.iter().filter(|field| field.opens()) { let misfit = |reason| StackCipherText::misfit(&field.name, reason); let Some((_, node)) = take(&mut row, &field.name) else { - return Err(misfit(absence( + let reason = absence( &row, &field.name, Reason::FieldMissing, Reason::FieldRepeated, - ))); + ); + // A `"null": "sql"` field left out of the record was null when + // it was stored: it opens as a null. + if reason == Reason::FieldMissing && field.null == Some(NullMode::Sql) { + let _ = absent.insert(field.name.clone()); + continue; + } + return Err(misfit(reason)); }; let CipherText::Map(mut outputs) = node else { return Err(misfit(Reason::OutputsNotMap)); @@ -2286,7 +2579,11 @@ fn record_row(mut row: Vec<(String, StackCipherText)>, plan: &Plan) -> Result, null: &str) -> Tree { + let mut entries = vec![("context", context), ("outputs", strings(outputs))]; + if let Some(ty) = ty { + entries.push(("type", s(ty))); + } + entries.push(("null", s(null))); + obj(entries) + } + + fn date(days: i32) -> Tree { + Tree::Typed(Logical::Date, Box::new(Tree::Int32(days))) + } + + /// The length of the sealed leaf under a stored field's `"c"`. + fn sealed_len(record: &StackCipherText, field: &str) -> usize { + let CipherText::Map(fields) = record else { + panic!("a record is a map"); + }; + let Some((_, CipherText::Map(outputs))) = fields.iter().find(|(name, _)| name == field) + else { + panic!("{field} is stored"); + }; + match outputs.iter().find(|(key, _)| key == "c") { + Some((_, CipherText::Single(leaf))) => leaf.to_bytes().len(), + other => panic!("{field}'s \"c\" is one leaf: {other:?}"), + } + } + + fn field_names(record: &StackCipherText) -> Vec<&str> { + match record { + CipherText::Map(fields) => fields.iter().map(|(name, _)| name.as_str()).collect(), + _ => panic!("a record is a map"), + } + } + + #[test] + fn the_plan_reads_how_each_field_stores_a_null() { + let parsed = plan(obj(vec![ + ( + "dob", + nullable(label("dob"), &["c"], Some("date"), "sealed"), + ), + ( + "age", + nullable(label("age"), &["c", "eq"], Some("uint32"), "sql"), + ), + ("nick", spec(label("nick"), &["c"])), + ])) + .expect("parses"); + let nulls: Vec<_> = parsed.fields().iter().map(FieldPlan::null).collect(); + assert_eq!(nulls, [Some(NullMode::Sealed), Some(NullMode::Sql), None]); + assert_eq!(NullMode::Sealed.name(), "sealed"); + assert_eq!(NullMode::Sql.name(), "sql"); + } + + /// Each field a null cannot be stored in as its `"null"` says is + /// refused when the plan is built, naming the field. + #[test] + fn a_field_that_cannot_store_its_null_so_is_refused() { + let cases = [ + ( + "an unknown mode", + nullable(label("f"), &["c"], Some("date"), "absent"), + Reason::UnknownNull, + ), + ( + "a null mode that is not text", + obj(vec![ + ("context", label("f")), + ("outputs", strings(&["c"])), + ("null", Tree::Bool(true)), + ]), + Reason::UnknownNull, + ), + ( + "a passthrough", + nullable(label("f"), &["passthrough"], None, "sql"), + Reason::NullPassthrough, + ), + ( + "a sealed null of text", + nullable(label("f"), &["c"], Some("string"), "sealed"), + Reason::SealedNullType, + ), + ( + "a sealed null of bytes", + nullable(label("f"), &["c"], Some("bytes"), "sealed"), + Reason::SealedNullType, + ), + ( + "a sealed null of no declared type", + nullable(label("f"), &["c"], None, "sealed"), + Reason::SealedNullType, + ), + ( + "a sealed null beside a term", + nullable(label("f"), &["c", "eq"], Some("int64"), "sealed"), + Reason::SealedNullIndex, + ), + ( + "a sealed null of a term alone", + nullable(label("f"), &["ore"], Some("int64"), "sealed"), + Reason::SealedNullIndex, + ), + ]; + for (what, field, reason) in cases { + let refused = plan(obj(vec![("f", field)])).err(); + assert!( + matches!( + &refused, + Some(Error::Plan { field: Some(f), reason: r }) if f == "f" && *r == reason + ), + "{what}: {refused:?}" + ); + } + } + + /// A null sealed under `"sealed"` is a leaf as long as a value of + /// the field's type, opens as a null, and costs a data key as a + /// value does. A boolean's null is the plain null, already as long. + #[tokio::test] + async fn a_sealed_null_is_as_long_as_a_value_and_opens_as_a_null() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(obj(vec![ + ( + "dob", + nullable(label("dob"), &["c"], Some("date"), "sealed"), + ), + ( + "score", + nullable(label("score"), &["c"], Some("float64"), "sealed"), + ), + ( + "flag", + nullable(label("flag"), &["c"], Some("bool"), "sealed"), + ), + ])) + .expect("parses"); + let present = obj(vec![ + ("dob", date(19_000)), + ("score", Tree::Float64(1.5)), + ("flag", Tree::Bool(true)), + ]); + let absent = obj(vec![ + ("dob", Tree::Null), + ("score", Tree::Null), + ("flag", Tree::Null), + ]); + check_source(absent.clone(), &plan).expect("a null fits a field that seals one"); + let with = seal(&keyset, present.clone(), &plan).await; + let without = seal(&keyset, absent.clone(), &plan).await; + for field in ["dob", "score", "flag"] { + assert_eq!( + sealed_len(&without, field), + sealed_len(&with, field), + "{field}: an absent value is as long as a present one" + ); + } + let again = seal(&keyset, absent.clone(), &plan).await; + check_record(again, &plan, None).expect("the stored nulls fit the plan"); + assert_eq!(open(&cipher, without, &plan).await, absent); + assert_eq!(open(&cipher, with, &plan).await, present); + } + + /// A null in a typed field with no `"null"` is still refused. + #[tokio::test] + async fn a_null_in_a_field_with_no_null_mode_is_refused() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = + plan(obj(vec![("dob", typed(label("dob"), &["c"], "date"))])).expect("parses"); + let refused = encrypt(&keyset, obj(vec![("dob", Tree::Null)]), &plan).err(); + assert!( + matches!(&refused, Some(Error::Source { field: Some(f), reason: Reason::FieldType }) if f == "dob"), + "{refused:?}" + ); + } + + /// Under `"sql"` a null, or a row with no entry for the field, is + /// left out of the stored record: no ciphertext, no term, no key. + /// The rest of the batch is one call, and every row opens with a + /// null for the field it left out. + #[tokio::test] + async fn a_sql_null_leaves_the_field_out_and_opens_as_a_null() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(obj(vec![ + ( + "age", + nullable(label("age"), &["c", "eq", "ore"], Some("uint32"), "sql"), + ), + ("email", spec(label("email"), &["c"])), + ])) + .expect("parses"); + let rows = Tree::Array(vec![ + obj(vec![("age", Tree::UInt32(34)), ("email", s("a@x"))]), + obj(vec![("age", Tree::Null), ("email", s("b@x"))]), + obj(vec![("email", s("c@x"))]), + obj(vec![("age", Tree::UInt32(35)), ("email", s("d@x"))]), + ]); + check_source(rows.clone(), &plan).expect("an absent field fits"); + let before = generates(&cipher); + let sealed = seal(&keyset, rows.clone(), &plan).await; + assert_eq!(generates(&cipher) - before, 1, "one call for every group"); + let CipherText::Sequence(stored) = &sealed else { + panic!("a batch is a sequence"); + }; + let names: Vec> = stored.iter().map(field_names).collect(); + assert_eq!( + names, + [ + vec!["age", "email"], + vec!["email"], + vec!["email"], + vec!["age", "email"], + ] + ); + let again = seal(&keyset, rows, &plan).await; + check_record(again, &plan, None).expect("the stored rows fit the plan"); + let opened = open(&cipher, sealed, &plan).await; + assert_eq!( + opened, + Tree::Array(vec![ + obj(vec![("age", Tree::UInt32(34)), ("email", s("a@x"))]), + obj(vec![("age", Tree::Null), ("email", s("b@x"))]), + obj(vec![("age", Tree::Null), ("email", s("c@x"))]), + obj(vec![("age", Tree::UInt32(35)), ("email", s("d@x"))]), + ]) + ); + } + + /// A row may leave out every field it seals; it is stored as an + /// empty record and opens with a null in each. + #[tokio::test] + async fn a_row_with_every_field_left_out_is_an_empty_record() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(obj(vec![( + "age", + nullable(label("age"), &["c", "eq"], Some("uint32"), "sql"), + )])) + .expect("parses"); + let sealed = seal(&keyset, obj(vec![("age", Tree::Null)]), &plan).await; + assert!(field_names(&sealed).is_empty(), "{sealed:?}"); + assert_eq!( + open(&cipher, sealed, &plan).await, + obj(vec![("age", Tree::Null)]) + ); + } + + /// A stored record with no entry for a field that has no `"null"` + /// is still refused: only a `"sql"` field may be left out. + #[tokio::test] + async fn a_stored_record_may_leave_out_only_a_sql_field() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(obj(vec![ + ("age", nullable(label("age"), &["c"], Some("uint32"), "sql")), + ("email", spec(label("email"), &["c"])), + ])) + .expect("parses"); + let sealed = seal( + &keyset, + obj(vec![("age", Tree::UInt32(3)), ("email", s("a@x"))]), + &plan, + ) + .await; + let without_email: Vec<_> = map(sealed) + .into_iter() + .filter(|(name, _)| name != "email") + .collect(); + let refused = check_record(CipherText::Map(without_email), &plan, None).err(); + assert!( + matches!(&refused, Some(Error::Record { field: Some(f), reason: Reason::FieldMissing }) if f == "email"), + "{refused:?}" + ); + } + } + mod given_a_typed_field { use super::*; @@ -5406,7 +5988,7 @@ mod tests { mod given_a_target_field { use super::*; use crate::dynamic::{NoTargets, TargetDescriptor, TargetError, TargetResolver}; - use crate::target::CallerContext; + use crate::target::{CallerContext, Decryption}; use crate::SealedValue; struct FakeEql; @@ -5977,14 +6559,14 @@ mod tests { let shape = plan.shape(); assert!( matches!( - shape_record(FieldValues::new(), Vec::new(), &shape), + shape_record(FieldValues::new(), Vec::new(), &shape, &Absent::new()), Err(crate::Error::ResponseShape) ), "a missing target value on the encrypt side" ); assert!( matches!( - open_record(FieldValues::new(), Vec::new(), &shape), + open_record(FieldValues::new(), Vec::new(), &shape, &Absent::new()), Err(crate::Error::ResponseShape) ), "a missing target value on the decrypt side" diff --git a/packages/stack-encrypt/src/dynamic/value_frame.rs b/packages/stack-encrypt/src/dynamic/value_frame.rs index 453fbc039..a25c36f3e 100644 --- a/packages/stack-encrypt/src/dynamic/value_frame.rs +++ b/packages/stack-encrypt/src/dynamic/value_frame.rs @@ -143,6 +143,33 @@ impl ValueFrame { Ok(Self(buffer)) } + /// The null, as a frame: what an absent value opens to. + pub(super) fn null() -> Self { + Self(Protected::new(vec![vitaminc_aead::tags::NULL])) + } + + /// A typed null of `ty`, as a frame: the leaf alone. Walked into the + /// encrypting sink it seals as a typed null, as long as a value of `ty` + /// (vitaminc 0.8); any other sink sees a null. The value model holds a + /// typed null of its own types only ([`model_null`]), so `ty` is one of + /// those a [`ValueKind`] names. + pub(super) fn typed_null(ty: FixedType) -> Self { + let mut leaf = Protected::new(Vec::new()); + vitaminc_aead::leaf::write_typed_null(ty, leaf.risky_inner_mut()); + Self(leaf) + } + + /// Whether the value is a null, plain or typed. + pub(super) fn is_null(&self) -> bool { + self.visit(super::term::OneScalar::new( + |ty: Option, value: Physical<'_>| match (ty, value) { + (None, Physical::Null) => Ok(()), + _ => Err(None), + }, + )) + .is_ok() + } + /// The frame bytes. Same chain-of-custody caveats as /// [`Protected::risky_ref`]: the bytes are plaintext, and the reference /// must not outlive its use. From 3c9084f4dd308b1bd4e25085a004266fa30c0853 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 19:51:31 +1100 Subject: [PATCH 02/14] feat(eql-bindings): a storage-only EQL type stores a None as long as a value A storage-only EQL v4 type (Date, Integer, Text, ...) carries no term, so an absent value has nothing that would show it but the length of `c`. Each now also derives EncryptFrom and DecryptInto for an Option of its plaintext, and the by-name dispatch the Go guest runs reads and opens a storage-only type through that Option: a value seals exactly the leaf the plaintext seals (Some walks the value itself), and a null seals as vitaminc 0.8's typed null, as long as a value, and opens as a null. A type with a term still refuses a null. eql-bindings' own Date, Timestamp and Decimal plaintexts declare their fixed width to vitaminc, so their None is padded too. The codegen emits the second plaintext attribute and the Option dispatch; the generated files are regenerated with `mise run types:generate`. Whether a field may hold a null is its plan's to say (#1298): the engine refuses a null for a target field without "null". Refs #1298. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql-bindings/src/encryption/targets.rs | 37 +++++++++ .../eql/crates/eql-bindings/src/v4/bigint.rs | 3 +- .../eql/crates/eql-bindings/src/v4/boolean.rs | 3 +- .../eql/crates/eql-bindings/src/v4/date.rs | 3 +- .../eql/crates/eql-bindings/src/v4/double.rs | 3 +- .../eql/crates/eql-bindings/src/v4/integer.rs | 3 +- .../eql/crates/eql-bindings/src/v4/numeric.rs | 3 +- .../crates/eql-bindings/src/v4/plaintext.rs | 20 +++-- .../eql/crates/eql-bindings/src/v4/real.rs | 3 +- .../crates/eql-bindings/src/v4/smallint.rs | 3 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 82 +++++++++++-------- .../eql/crates/eql-bindings/src/v4/text.rs | 3 +- .../crates/eql-bindings/src/v4/timestamp.rs | 3 +- .../eql/crates/eql-codegen/src/bindings.rs | 12 +++ .../eql/crates/eql-codegen/src/targets.rs | 36 ++++++-- .../tests/encryption/tests/typed_queries.rs | 74 +++++++++++++++++ .../encryption/tests/v4_date_time_numeric.rs | 80 ++++++++++++++++++ 17 files changed, 313 insertions(+), 58 deletions(-) diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 79912347a..6a20a7e5c 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -449,6 +449,7 @@ mod sealed { impl Sealed for crate::v4::plaintext::Date {} impl Sealed for crate::v4::plaintext::Timestamp {} impl Sealed for crate::v4::plaintext::Decimal {} + impl Sealed for Option {} } /// A Rust plaintext an EQL type is produced from, read out of and written @@ -544,6 +545,42 @@ mod logical { ); } +/// The plaintext of a storage-only type, which may be absent: what the +/// generated dispatch reads a storage-only type's value as, and opens it +/// into. A null seals as `None`, a typed null of the plaintext's type, as +/// long as a value when the type is fixed width (vitaminc 0.8), and opens +/// as a null; a value seals and opens exactly as the plaintext does, since +/// `Some(v)` walks `v` itself. A storage-only type has no term to derive +/// from a null. Whether a field may hold a null is the plan's to decide +/// (its `"null"`), not the type's. +impl Plaintext for Option { + const KIND: ValueKind = S::KIND; + + fn from_frame(target: &'static str, value: &ValueFrame) -> Result { + if value.kind().is_some() { + return S::from_frame(target, value).map(Some); + } + // No kind: a null, which reads as `None`, or something no plaintext + // is (an `undefined`, a passthrough), refused as `S` refuses it. + match value.visit(Self::sink()) { + Ok(None) => Ok(None), + _ => Err(TargetError::Plaintext { + target, + expected: Self::KIND, + found: None, + }), + } + } + + fn into_frame(self) -> Result { + match self { + Some(value) => value.into_frame(), + None => ValueFrame::from_bytes(&[vitaminc_aead::tags::NULL]) + .map_err(|error| Error::Other(Box::new(error))), + } + } +} + /// Run a target's own plan over one runtime value and resolve to the EQL /// value's JSON bytes. The generated dispatch calls this with the type and /// its plaintext; it is the one place the typed `encrypt_as` is reached from diff --git a/packages/eql/crates/eql-bindings/src/v4/bigint.rs b/packages/eql/crates/eql-bindings/src/v4/bigint.rs index 740d6c88d..3fedc4b73 100644 --- a/packages/eql/crates/eql-bindings/src/v4/bigint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/bigint.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Bigint { diff --git a/packages/eql/crates/eql-bindings/src/v4/boolean.rs b/packages/eql/crates/eql-bindings/src/v4/boolean.rs index 02be819d4..7b3dd2237 100644 --- a/packages/eql/crates/eql-bindings/src/v4/boolean.rs +++ b/packages/eql/crates/eql-bindings/src/v4/boolean.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL boolean reference](https://cipherstash.com/docs/reference/eql/booleans) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`bool`](bool). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`bool`](bool). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = bool)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Boolean { diff --git a/packages/eql/crates/eql-bindings/src/v4/date.rs b/packages/eql/crates/eql-bindings/src/v4/date.rs index d14e2cee0..8651b9126 100644 --- a/packages/eql/crates/eql-bindings/src/v4/date.rs +++ b/packages/eql/crates/eql-bindings/src/v4/date.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL date reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Date)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Date { diff --git a/packages/eql/crates/eql-bindings/src/v4/double.rs b/packages/eql/crates/eql-bindings/src/v4/double.rs index f2e00f4d9..4b694e237 100644 --- a/packages/eql/crates/eql-bindings/src/v4/double.rs +++ b/packages/eql/crates/eql-bindings/src/v4/double.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = f64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Double { diff --git a/packages/eql/crates/eql-bindings/src/v4/integer.rs b/packages/eql/crates/eql-bindings/src/v4/integer.rs index 8fcb2c945..e7f59f7b5 100644 --- a/packages/eql/crates/eql-bindings/src/v4/integer.rs +++ b/packages/eql/crates/eql-bindings/src/v4/integer.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Integer { diff --git a/packages/eql/crates/eql-bindings/src/v4/numeric.rs b/packages/eql/crates/eql-bindings/src/v4/numeric.rs index d9ead5040..197fccbc9 100644 --- a/packages/eql/crates/eql-bindings/src/v4/numeric.rs +++ b/packages/eql/crates/eql-bindings/src/v4/numeric.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL numeric reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Decimal)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Numeric { diff --git a/packages/eql/crates/eql-bindings/src/v4/plaintext.rs b/packages/eql/crates/eql-bindings/src/v4/plaintext.rs index 7af87f6ad..d3eec609c 100644 --- a/packages/eql/crates/eql-bindings/src/v4/plaintext.rs +++ b/packages/eql/crates/eql-bindings/src/v4/plaintext.rs @@ -67,7 +67,7 @@ use std::fmt; use stack_encrypt::Encrypt; use vitaminc_aead::plaintext::depth; use vitaminc_aead::plaintext::kinds::{timestamp_nanos, MAX_DECIMAL_SCALE}; -use vitaminc_aead::{FromScalar, Int, Logical, Physical, Sink, TimeUnit}; +use vitaminc_aead::{FixedType, FromScalar, Int, Logical, Physical, Sink, TimeUnit, Width}; use zeroize::{Zeroize, ZeroizeOnDrop}; const NANOS_PER_SECOND: u32 = 1_000_000_000; @@ -310,10 +310,16 @@ wiped!( /// it is, the same scalar the chrono and rust_decimal types describe, so a /// value converted from one of those seals the same leaf. The owned form /// visits through the borrowed one and is then dropped, and wiped. +/// +/// Each is fixed width, so `None` of one seals as a typed null as long as a +/// value (vitaminc 0.8), and an `Option` of one reads it back: the type's +/// own zero (the epoch, a zero decimal) is a value it reads, which is what +/// `Decrypt::reads_null` asks of it. macro_rules! source { - ($($ty:ident => |$value:ident| $scalar:expr),+ $(,)?) => {$( + ($($ty:ident ($fixed:expr) => |$value:ident| $scalar:expr),+ $(,)?) => {$( impl Encrypt for &$ty { const DEPTH: Option = depth::SCALAR; + const FIXED: Option = $fixed; fn visit(self, sink: S) -> Result { let $value = self; let (logical, int) = $scalar; @@ -323,6 +329,7 @@ macro_rules! source { impl Encrypt for $ty { const DEPTH: Option = depth::SCALAR; + const FIXED: Option = $fixed; fn visit(self, sink: S) -> Result { (&self).visit(sink) } @@ -330,9 +337,12 @@ macro_rules! source { )+}; } source!( - Date => |date| (Logical::Date, Int::I32(date.days)), - Timestamp => |instant| (Logical::Timestamp(TimeUnit::Nanos), Int::I128(instant.count())), - Decimal => |decimal| (Logical::Decimal { scale: decimal.scale }, Int::I128(decimal.mantissa)), + Date (FixedType::new(Some(Logical::Date), Width::I32)) + => |date| (Logical::Date, Int::I32(date.days)), + Timestamp (FixedType::new(Some(Logical::Timestamp(TimeUnit::Nanos)), Width::I128)) + => |instant| (Logical::Timestamp(TimeUnit::Nanos), Int::I128(instant.count())), + Decimal (FixedType::new(Some(Logical::Decimal { scale: 0 }), Width::I128)) + => |decimal| (Logical::Decimal { scale: decimal.scale }, Int::I128(decimal.mantissa)), ); /// A date reads exactly a date scalar, at any integer width, in range. diff --git a/packages/eql/crates/eql-bindings/src/v4/real.rs b/packages/eql/crates/eql-bindings/src/v4/real.rs index 945374401..5651100ca 100644 --- a/packages/eql/crates/eql-bindings/src/v4/real.rs +++ b/packages/eql/crates/eql-bindings/src/v4/real.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = f32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Real { diff --git a/packages/eql/crates/eql-bindings/src/v4/smallint.rs b/packages/eql/crates/eql-bindings/src/v4/smallint.rs index 503f7bf7b..963a8f9be 100644 --- a/packages/eql/crates/eql-bindings/src/v4/smallint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/smallint.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i16)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Smallint { diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index feddec846..20ed10e19 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -659,9 +659,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( plaintext: ValueFrame, ) -> Result, K>, TargetError> { match name { - "Integer" => { - run_target::("Integer", keyset, column, plaintext) - } + "Integer" => run_target::, K>( + "Integer", keyset, column, plaintext, + ), "IntegerEq" => { run_target::("IntegerEq", keyset, column, plaintext) } @@ -683,9 +683,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Smallint" => { - run_target::("Smallint", keyset, column, plaintext) - } + "Smallint" => run_target::, K>( + "Smallint", keyset, column, plaintext, + ), "SmallintEq" => run_target::( "SmallintEq", keyset, @@ -711,7 +711,7 @@ pub(crate) fn encrypt_named<'a, K: 'static>( plaintext, ), "Bigint" => { - run_target::("Bigint", keyset, column, plaintext) + run_target::, K>("Bigint", keyset, column, plaintext) } "BigintEq" => { run_target::("BigintEq", keyset, column, plaintext) @@ -731,9 +731,11 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Date" => run_target::("Date", keyset, column, plaintext), + "Date" => { + run_target::, K>("Date", keyset, column, plaintext) + } "DateEq" => run_target::("DateEq", keyset, column, plaintext), - "Timestamp" => run_target::( + "Timestamp" => run_target::, K>( "Timestamp", keyset, column, @@ -745,16 +747,18 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Numeric" => { - run_target::("Numeric", keyset, column, plaintext) - } + "Numeric" => run_target::, K>( + "Numeric", keyset, column, plaintext, + ), "NumericEq" => run_target::( "NumericEq", keyset, column, plaintext, ), - "Text" => run_target::("Text", keyset, column, plaintext), + "Text" => { + run_target::, K>("Text", keyset, column, plaintext) + } "TextEq" => { run_target::("TextEq", keyset, column, plaintext) } @@ -788,10 +792,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Boolean" => { - run_target::("Boolean", keyset, column, plaintext) + "Boolean" => run_target::, K>( + "Boolean", keyset, column, plaintext, + ), + "Real" => { + run_target::, K>("Real", keyset, column, plaintext) } - "Real" => run_target::("Real", keyset, column, plaintext), "RealEq" => run_target::("RealEq", keyset, column, plaintext), "RealOrdOre" => { run_target::("RealOrdOre", keyset, column, plaintext) @@ -803,7 +809,7 @@ pub(crate) fn encrypt_named<'a, K: 'static>( run_target::("RealOrdOpe", keyset, column, plaintext) } "Double" => { - run_target::("Double", keyset, column, plaintext) + run_target::, K>("Double", keyset, column, plaintext) } "DoubleEq" => { run_target::("DoubleEq", keyset, column, plaintext) @@ -835,9 +841,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( stored: &[u8], ) -> Result, TargetError> { match name { - "Integer" => { - open_target::("Integer", opener, column, stored) - } + "Integer" => open_target::, K>( + "Integer", opener, column, stored, + ), "IntegerEq" => { open_target::("IntegerEq", opener, column, stored) } @@ -856,9 +862,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Smallint" => { - open_target::("Smallint", opener, column, stored) - } + "Smallint" => open_target::, K>( + "Smallint", opener, column, stored, + ), "SmallintEq" => { open_target::("SmallintEq", opener, column, stored) } @@ -880,7 +886,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Bigint" => open_target::("Bigint", opener, column, stored), + "Bigint" => { + open_target::, K>("Bigint", opener, column, stored) + } "BigintEq" => { open_target::("BigintEq", opener, column, stored) } @@ -899,9 +907,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Date" => open_target::("Date", opener, column, stored), + "Date" => open_target::, K>("Date", opener, column, stored), "DateEq" => open_target::("DateEq", opener, column, stored), - "Timestamp" => open_target::( + "Timestamp" => open_target::, K>( "Timestamp", opener, column, @@ -913,16 +921,18 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Numeric" => { - open_target::("Numeric", opener, column, stored) - } + "Numeric" => open_target::, K>( + "Numeric", opener, column, stored, + ), "NumericEq" => open_target::( "NumericEq", opener, column, stored, ), - "Text" => open_target::("Text", opener, column, stored), + "Text" => { + open_target::, K>("Text", opener, column, stored) + } "TextEq" => open_target::("TextEq", opener, column, stored), "TextMatch" => { open_target::("TextMatch", opener, column, stored) @@ -945,10 +955,10 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "TextSearch" => { open_target::("TextSearch", opener, column, stored) } - "Boolean" => { - open_target::("Boolean", opener, column, stored) - } - "Real" => open_target::("Real", opener, column, stored), + "Boolean" => open_target::, K>( + "Boolean", opener, column, stored, + ), + "Real" => open_target::, K>("Real", opener, column, stored), "RealEq" => open_target::("RealEq", opener, column, stored), "RealOrdOre" => { open_target::("RealOrdOre", opener, column, stored) @@ -957,7 +967,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "RealOrdOpe" => { open_target::("RealOrdOpe", opener, column, stored) } - "Double" => open_target::("Double", opener, column, stored), + "Double" => { + open_target::, K>("Double", opener, column, stored) + } "DoubleEq" => { open_target::("DoubleEq", opener, column, stored) } diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index af5544826..13cc191bb 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Text { diff --git a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs index 19bd93cf2..072a559f7 100644 --- a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs +++ b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL timestamp reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Timestamp)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Timestamp { diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index cb5011b29..17bd9654c 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -730,6 +730,13 @@ fn encryption_attrs( " Plaintext input: {named}. Implements `EncryptFrom<{shown}>` to build {terms} \ query terms. Query operands do not decrypt." ) + } else if domain.terms.is_empty() { + format!( + " Plaintext input and decrypted output: {named}. Implements \ + `EncryptFrom<{shown}>` and `DecryptInto<{shown}>`, and both for \ + `Option<{shown}>`: a storage-only type carries no term, so it may store a \ + `None`, sealed as long as a value." + ) } else { format!( " Plaintext input and decrypted output: {named}. Implements \ @@ -738,6 +745,10 @@ fn encryption_attrs( } }; let plaintext = plaintext_doc(&format!("[`{shown}`]({rust})")); + // A storage-only type derives no term, so an absent value has nothing + // it cannot store: it may be sealed from an `Option` too (#1298). + let nullable = (!query && domain.terms.is_empty()) + .then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); // The twin of an optional-term query is built by its projections // (`render_query_projections`), one per index, so it carries no derive. if query && v4_optional_query_terms(domain) { @@ -769,6 +780,7 @@ fn encryption_attrs( #[derive(stack_encrypt::EncryptFrom)] #decrypt #[stash(plaintext = #plaintext_ty)] + #nullable }, quote!(#[stash(default)]), quote!(#[stash(context_field)]), diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 6019c73b1..0b3e9935e 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -425,10 +425,21 @@ fn render_targets_from(rows: &[Row]) -> String { let source = format_ident!("{}", rust.rsplit("::").next().unwrap_or(rust)); (name.clone(), quote!(super::#module::#ty), quote!(#source)) }; + // A storage-only type (no query twin, so no term) seals and opens an + // `Option` of its plaintext: a value exactly as the plaintext does, and + // a null as a null (#1298). Whether a field may hold one is its plan's. + let stored_source = |r: &Row, source: TokenStream| { + if r.query.is_none() { + quote!(Option<#source>) + } else { + source + } + }; let encrypt_arms: TokenStream = producible .iter() .map(|r| { let (name, ty, source) = arm(r, &r.name); + let source = stored_source(r, source); quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), } }) .collect(); @@ -436,6 +447,7 @@ fn render_targets_from(rows: &[Row]) -> String { .iter() .map(|r| { let (name, ty, source) = arm(r, &r.name); + let source = stored_source(r, source); quote! { #name => open_target::<#ty, #source, K>(#name, opener, column, stored), } }) .collect(); @@ -824,15 +836,22 @@ mod tests { let module = family.name; let (_, plaintext) = plaintext_of(family.name).expect("a derived family's plaintext"); let plaintext = plaintext.rsplit("::").next().unwrap(); + // A storage-only type stores and opens an `Option` of its + // plaintext; its query arm, which it has none of, would not. + let stored = if domain.terms.is_empty() { + format!("Option<{plaintext}>") + } else { + plaintext.to_owned() + }; let arm = |helper: &str, ty: &str| { + let source = if ty == name { &stored } else { plaintext }; format!( - "\"{name}\" => {{ {helper}::(\"{name}\", " + "\"{name}\" => {{ {helper}::(\"{name}\", " ) }; let flat_arm = |helper: &str, ty: &str| { - format!( - "\"{name}\" => {helper}::(\"{name}\", " - ) + let source = if ty == name { &stored } else { plaintext }; + format!("\"{name}\" => {helper}::(\"{name}\", ") }; // rustfmt breaks long arms at different points, so compare with // all whitespace gone. @@ -1073,8 +1092,9 @@ mod tests { } /// A producible type with no query twin — the storage-only `Text` — - /// renders encrypt and decrypt arms and no query arm: the query - /// dispatch's fall-through refuses it as answering no query. + /// renders encrypt and decrypt arms over an `Option` of its plaintext, + /// so it stores a null (#1298), and no query arm: the query dispatch's + /// fall-through refuses it as answering no query. #[test] fn a_producible_type_without_a_query_twin_gets_no_query_arm() { let rows = rows(); @@ -1089,12 +1109,12 @@ mod tests { .join(" "); let arms_for = |helper: &str| { out.matches(&format!( - "\"Text\" => {{ {helper}::(\"Text\"," + "\"Text\" => {{ {helper}::, K>(\"Text\"," )) .count() + out .matches(&format!( - "\"Text\" => {helper}::(\"Text\"," + "\"Text\" => {helper}::, K>(\"Text\"," )) .count() }; diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index 1f5ec021d..eab78044e 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -540,3 +540,77 @@ async fn an_extended_typed_query_is_refused_before_any_keyset() { .unwrap(); assert_eq!(calls.lock().unwrap().loads, 1); } + +/// A record field naming a storage-only EQL type may store a null as the +/// plan says (#1298), through the resolver the Go guest installs: +/// `"sealed"` stores an EQL value whose `c` is a typed null, as long as a +/// value's, and `"sql"` stores nothing for the field. Both open as a null. +#[tokio::test] +async fn a_record_stores_a_null_in_an_eql_field_as_its_plan_says() { + use dynamic::{NullMode, Scope}; + use stack_encrypt::CipherText; + use std::collections::BTreeMap; + + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let context = |column: &str| { + NonEmpty::new(ContextPiece::List(vec![ + ContextPiece::Text("users".into()), + ContextPiece::Text(column.to_owned().into()), + ])) + .unwrap() + }; + let plan = record::Plan::new_with( + vec![ + FieldPlan::with_target("a_dob", context("dob"), "Date") + .unwrap() + .with_null(NullMode::Sealed), + FieldPlan::with_target("b_born", context("born"), "DateEq") + .unwrap() + .with_null(NullMode::Sql), + ], + &Eql, + ) + .unwrap(); + let day = chrono::NaiveDate::from_ymd_opt(1990, 5, 17); + let rows = |dob, born| BTreeMap::from([("a_dob", dob), ("b_born", born)]); + let source = vec![rows(day, day), rows(None, None)]; + let sealed = record::encrypt_with(&keyset, frame(&source), &plan, &Eql) + .unwrap() + .await + .unwrap(); + + // The EQL value's size, as the stored record carries it. + let eql_len = |row: &stack_encrypt::StackCipherText, field: &str| -> Option { + let CipherText::Map(fields) = row else { + panic!("a row is a map") + }; + let (_, CipherText::Map(outputs)) = fields.iter().find(|(name, _)| name == field)? else { + panic!("{field} is a map of outputs"); + }; + match outputs.iter().find(|(key, _)| key == "eql") { + Some((_, CipherText::Passthrough(payload))) => Some(payload.0.len()), + other => panic!("{field}'s EQL value is a passthrough: {other:?}"), + } + }; + let CipherText::Sequence(stored) = &sealed else { + panic!("a batch is a sequence"); + }; + assert_eq!( + eql_len(&stored[1], "a_dob"), + eql_len(&stored[0], "a_dob"), + "a sealed null is as long as a value" + ); + assert!(eql_len(&stored[0], "b_born").is_some()); + assert_eq!( + eql_len(&stored[1], "b_born"), + None, + "a sql null stores nothing" + ); + + let opened = record::decrypt_with(Scope::Client(&cipher), sealed, &plan, None, &Eql) + .unwrap() + .await + .unwrap(); + assert_eq!(opened.risky_ref(), frame(&source).risky_ref()); +} diff --git a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs index 466e7377c..c451c4bf6 100644 --- a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs +++ b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs @@ -907,3 +907,83 @@ fn postgres_stores_and_queries_ledger_rows_the_go_sdk_sealed() { assert_eq!(ids, expected, "= over Go-sealed {column_name}"); } } + +/// The length of the sealed leaf a stored `c` holds. +fn leaf_len(c: &Ciphertext) -> usize { + match c.clone().read(NativeLeaf).unwrap() { + stack_encrypt::CipherText::Single(leaf) => leaf.to_bytes().len(), + _ => panic!("a stored c is one leaf"), + } +} + +/// A storage-only type stores an absent value (#1298): `None` seals as a +/// typed null as long as a value, so the column does not show which rows +/// are absent, and opens as `None`, on the typed path and by name. A value +/// sealed from `Some` is the leaf the plaintext alone seals, so each opens +/// the other's. +#[tokio::test] +async fn a_storage_only_type_stores_a_none_as_long_as_a_value() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let value = Date::from(day("2024-02-29")); + + let present: date::Date = keyset.encrypt_as(&value, column()).await.unwrap(); + let absent: date::Date = keyset.encrypt_as(&None::, column()).await.unwrap(); + assert_eq!( + leaf_len(&absent.c), + leaf_len(&present.c), + "as long as a value" + ); + let opened: Option = cipher + .decrypt_as(absent.clone(), column().into()) + .await + .unwrap(); + assert!(opened.is_none()); + let refused: Result = cipher.decrypt_as(absent.clone(), column().into()).await; + assert!(refused.is_err(), "a None does not open as a value"); + + let some: date::Date = keyset + .encrypt_as(&Some(value.clone()), column()) + .await + .unwrap(); + let opened: Date = cipher.decrypt_as(some, column().into()).await.unwrap(); + assert_eq!( + NaiveDate::from(&opened), + day("2024-02-29"), + "Some seals the plaintext's own leaf" + ); + let opened: Option = cipher.decrypt_as(present, column().into()).await.unwrap(); + assert_eq!( + opened.as_ref().map(NaiveDate::from), + Some(day("2024-02-29")), + "and an Option opens a value" + ); + + let null = ValueFrame::from_bytes(&[0x00]).unwrap(); + let named: date::Date = by_name(&keyset, "Date", null.clone()).await; + assert_eq!(leaf_len(&named.c), leaf_len(&absent.c), "by name too"); + let opened = targets::decrypt( + "Date", + &cipher, + &label(), + &serde_json::to_vec(&named).unwrap(), + ) + .unwrap() + .await + .unwrap(); + assert!(same(&opened, &null), "opens as a null by name"); + + // A type with a term has no null to store: its term would show it. + assert!(matches!( + refused_null(&keyset, "DateEq"), + TargetError::Plaintext { .. } + )); +} + +fn refused_null( + keyset: &stack_encrypt::KeysetCipher<'_, common::Observed>, + name: &str, +) -> TargetError { + let null = ValueFrame::from_bytes(&[0x00]).unwrap(); + refused(targets::encrypt(name, keyset, &label(), null)) +} From 089a9d97e8419cfd9cc948fa0a09ef8b6555bb12 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 20:04:24 +1100 Subject: [PATCH 03/14] feat(golang): an optional sealed field, declared over a pointer A sealed field may now be declared over a pointer to one scalar (`DOB *encrypt.Date` with `stash:"dob,encrypt_into=Date"`, which #1293 reproduced being refused as a composite), and Decrypt gives nil back for a nil (#1298). A new tag part, `null=sealed` or `null=sql`, says how nil is stored, and reaches the engine as the data plan's "null": - sealed: nil is sealed as a typed null as long as a value, so the column does not show which rows are absent. stashgen picks it for a field with no index whose type has a fixed width, sealed with encrypt or into a storage-only EQL type. - sql: nothing is stored, so the field's columns hold SQL NULL and its member in the generated type is a pointer. A field with an index, or of text or bytes, has to say so: a null has no term yet, and text shows its length, so stashgen will not choose a visible NULL silently. gensupport gains Declaration.Null, Deref and GetPtr; the record model and the encrypt record path carry the mode and accept a SQLNull field left out. A stored null read into a field that is not a pointer is a Decrypt error, from the engine's kind check. -model does not map a null=sql field yet, and stashgen says so. The reproduction from #1293 is folded into cmd/stashgen's tests, with the guest-built skip the other engine test has. Closes #1293. Refs #1298. Co-authored-by: Jonathon Walker <1989282+jonathon-walker@users.noreply.github.com> Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql-bindings-optional-storage-only.md | 5 + languages/golang/CHANGELOG.md | 17 ++ languages/golang/cmd/stashgen/README.md | 23 +- languages/golang/cmd/stashgen/main_test.go | 42 +++ languages/golang/encrypt/ciphertext.go | 4 +- languages/golang/encrypt/gensupport/codec.go | 42 ++- .../golang/encrypt/gensupport/declaration.go | 46 +++- .../internal/testusers/birthday_stash.go | 94 +++++++ .../encrypt/internal/testusers/people.go | 28 ++ .../internal/testusers/person_stash.go | 260 ++++++++++++++++++ languages/golang/encrypt/null_test.go | 93 +++++++ languages/golang/encrypt/records.go | 12 +- languages/golang/encrypt/values.go | 6 +- languages/golang/internal/record/record.go | 36 ++- languages/golang/stashgen/declaration.go | 33 +++ languages/golang/stashgen/emit.go | 145 +++++++--- languages/golang/stashgen/engine.go | 2 +- languages/golang/stashgen/read.go | 69 ++++- languages/golang/stashgen/refusal_test.go | 12 + languages/golang/stashgen/tag.go | 21 +- .../stubsdk/encrypt/gensupport/gensupport.go | 22 ++ packages/stack-encrypt/CHANGELOG.md | 17 ++ 22 files changed, 981 insertions(+), 48 deletions(-) create mode 100644 .changeset/eql-bindings-optional-storage-only.md create mode 100644 languages/golang/encrypt/internal/testusers/birthday_stash.go create mode 100644 languages/golang/encrypt/internal/testusers/people.go create mode 100644 languages/golang/encrypt/internal/testusers/person_stash.go create mode 100644 languages/golang/encrypt/null_test.go diff --git a/.changeset/eql-bindings-optional-storage-only.md b/.changeset/eql-bindings-optional-storage-only.md new file mode 100644 index 000000000..67e8fbc35 --- /dev/null +++ b/.changeset/eql-bindings-optional-storage-only.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": patch +--- + +`eql-bindings` (the Rust crate released with this package): a storage-only EQL v4 type (`Date`, `Integer`, `Text`, and the other types with no term) now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads and opens these types through the `Option`, so a null frame encrypts and a stored null decrypts to a null; a type with a term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index ac917da20..4c3a02a37 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -10,6 +10,23 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ## [Unreleased] +### Added + +- **An optional field: a sealed field declared over a pointer (#1298).** + `DOB *encrypt.Date` with `stash:"dob,encrypt_into=Date"` generates (it + was refused as a composite, #1293), and `Decrypt` gives `nil` back for a + `nil`. The new `null=` tag part says how `nil` is stored: + `null=sealed` seals it as a null as long as a value of the field's type, + so the column does not show which rows have no value; `null=sql` stores + nothing, so the field's columns are SQL `NULL` and its member in the + generated type is a pointer. `stashgen` picks `null=sealed` for a field + with no index whose type has a fixed width (a bool, a number, + `encrypt.Date`, `time.Time`, `encrypt.Decimal`, the 128-bit integers), + and asks for `null=sql` to be written on any other: a null has no index + term yet, and text shows its length. A stored null read into a field that + is not a pointer is an error from `Decrypt`. `-model` does not map a + `null=sql` field yet. + ### Breaking - **Every order term changes: re-derive stored `ore`, `ope` and diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 3162fe90f..cb4c99fe3 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -79,6 +79,7 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | | `stash:"-"` | leave the field out | | `` _ struct{} `stash:"context=documents,opaque"` `` | seal the struct as one value | +| `stash:"born,encrypt_into=DateEq,null=sql"` | on a field declared over a pointer (`*encrypt.Date`): how it stores `nil`, `null=sealed` or `null=sql`; see "Optional fields" | A struct has one context: either a `_ struct{}` field with `context=`, or one field with `context_field`, not both. A `context_field` is a `string` whose value is a label such as `"tenants/acme"`, segments separated by `/`. @@ -109,6 +110,24 @@ These words are the same as the Rust API's words for the same behaviour, with tw `opaque` seals the whole struct as one `bytes` field holding a JSON document (see "What crosses the binding"). `json` names the JSON index, which the Rust API does not declare yet; this build refuses it. +### Optional fields + +A sealed field declared over a pointer to one scalar (`*encrypt.Date`, `*int32`, `*string`, a pointer to a type defined over a scalar) is optional: `nil` is a value it may hold, and `Decrypt` gives it back as `nil`. +`null=` says how the field stores `nil`. There are two ways, and they differ in what a reader of the database can see: + +| `null=` | Stored for `nil` | What a reader of the column sees | +|---|---|---| +| `sealed` | a sealed null, as long as a sealed value of the field's type | nothing: an absent value looks like any other | +| `sql` | nothing: the field's columns are `NULL`, and its member in the generated type is `nil` | which rows have no value | + +A sealed null hides that the value is absent only where its length matches a value's and nothing beside it shows it, so it is the default for a field with no index whose type has a fixed width: a `bool`, a number, `encrypt.Date`, `time.Time`, `encrypt.Decimal`, `encrypt.Int128` and `encrypt.Uint128`, sealed with `encrypt` or into an EQL type with no index (`Date`, `Integer`, `Boolean`, ...). +A field with an index, and a field of text or bytes, has to write `null=sql`: a null has no index term yet, and text shows its length, so a sealed null would hide nothing there, and `stashgen` will not choose a visible `NULL` for you. +`null=sealed` on such a field stops `stashgen` with the reason. + +A `null=sql` field's member in the generated type is a pointer (`*eql.DateEq`, `*EncryptedUserScore`), `nil` when the value was; a `null=sealed` field's is the same type as a non-optional field's, and its column is never `NULL`. +A stored null read into a field that is not a pointer is an error from `Decrypt`, naming the field, never a zero value. +`-model` does not map a `null=sql` field yet, and `stashgen` stops if it is asked to. + An embedded struct of your own adds its tagged fields to the outer struct. An embedded struct from another package takes one tag for all of its fields: `stash:",passthrough"` stores them as they are, and `stash:"-"` leaves them out. Tags for other libraries, such as `gorm`, `db` and `json`, are copied to the same field of the generated type. @@ -254,10 +273,10 @@ Generated code sends the engine every sealed field with its value, under the dec A passthrough field stays on the host: the engine does nothing to it a program could observe, and the FFI codec cannot carry every Go type a program stores beside a ciphertext (a `driver.Valuer`, a `sql.NullTime`). An `opaque` struct crosses as one JSON document and is one column, decoded back into the exact Go types the struct declares; a field may be any type `encoding/json` carries both ways (a scalar or a type defined over one, `[]byte`, slices, maps with string or integer keys, pointers, structs whose fields are all exported, `time.Time`). A float in an opaque struct must be finite: `encoding/json` refuses NaN and the infinities, so `Encrypt` fails for the batch with `encrypt.ErrEncoding`. A sealed float field outside an opaque struct carries them. -A sealed field outside an opaque struct is one scalar, or a type defined over one; a struct, slice or map field seals only inside an opaque struct. +A sealed field outside an opaque struct is one scalar, a type defined over one, or a pointer to either (see "Optional fields"); a struct, slice or map field seals only inside an opaque struct. The scalars are `string`, `[]byte`, `bool`, every integer type (each sealed at its own width, `int` and `uint` as 64-bit), `encrypt.Int128` and `encrypt.Uint128`, `float32`, `float64`, `encrypt.Decimal`, `encrypt.Date` and `time.Time` (a timestamp). The wire type `stashgen` writes for each is the field's own: `int8` is `gensupport.Int8`, `time.Time` is `gensupport.Timestamp`, and so on. An order index (`ore`, `ope`) on a decimal, date or timestamp is refused: those order terms are not supported yet (#1236). -The zero `encrypt.Date` is not a date: a date field must be set, and `Encrypt` refuses a zero one with `encrypt.ErrEncoding`, naming the field. A sealed field cannot be a pointer (`*encrypt.Date` is refused as a composite), so an optional date has no sealed form yet: keep it in an opaque struct, or store a sentinel date. +The zero `encrypt.Date` is not a date: a date field must be set, and `Encrypt` refuses a zero one with `encrypt.ErrEncoding`, naming the field. An optional date is a `*encrypt.Date` field (see "Optional fields"). A nil `[]byte` in a sealed field, or a nil slice of a type defined over `[]byte`, comes back from `Decrypt` as an empty, non-nil slice; inside an opaque struct, nil comes back as nil. A protobuf message with a `oneof` cannot be generated from a policy: protoc-gen-go puts its members in wrapper types, not in the message struct, and `Generate` refuses it. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 84ab1ddb9..a106a4771 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -119,6 +119,48 @@ func TestRunFlags(t *testing.T) { } } +// A storage-only EQL type over a pointer (#1293): the generator takes +// *encrypt.Date as it takes encrypt.Date, and stores nil as a sealed null, +// as long as a date, since Date carries no index. From the reproduction in +// #1293. Skips when the guest is not built, as the engine decides what an +// EQL type is. +func TestRunGeneratesNullableDOB(t *testing.T) { + checker, err := encrypt.NewChecker(context.Background()) + if err != nil { + t.Skip(err) + } + _ = checker.Close() + for _, dateType := range []string{"encrypt.Date", "*encrypt.Date"} { + t.Run(dateType, func(t *testing.T) { + src := `package people + +import "github.com/cipherstash/stack/languages/golang/encrypt" + +type Person struct { + _ struct{} ` + "`stash:\"context=people\"`" + ` + DOB ` + dateType + " `stash:\"dob,encrypt_into=Date\"`" + ` +} +` + dir := writeModule(t, map[string]string{"model.go": src}) + var stdout, stderr bytes.Buffer + if code := run([]string{"-type", "Person"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 0 { + t.Fatalf("exit %d, want 0\n%s", code, stderr.String()) + } + written, err := os.ReadFile(filepath.Join(dir, "person_stash.go")) + if err != nil { + t.Fatal(err) + } + sealedNull := strings.Contains(string(written), `Null("dob", gensupport.NullSealed)`) + if pointer := strings.HasPrefix(dateType, "*"); sealedNull != pointer { + t.Fatalf("a %s field stores nil as a sealed null: %v, want %v\n%s", dateType, sealedNull, pointer, written) + } + if !strings.Contains(string(written), "DOB eql.Date") { + t.Fatalf("a sealed null keeps the column's type, eql.Date:\n%s", written) + } + }) + } +} + // The real engine: the embedded guest, which the command links with the EQL // types, so a struct with encrypt_into=TextEq generates, and a type the // engine cannot produce yet is refused with nothing written. Skips when the diff --git a/languages/golang/encrypt/ciphertext.go b/languages/golang/encrypt/ciphertext.go index 7d70475d8..77816f727 100644 --- a/languages/golang/encrypt/ciphertext.go +++ b/languages/golang/encrypt/ciphertext.go @@ -15,7 +15,9 @@ import ( // vitaminc-encrypt and must never scan or marshal where one belongs. // // A sealed field is one scalar — a string, a number, a bool or a []byte, or -// a type defined over one — and seals as vitaminc's tagged leaf, which a +// a type defined over one, or a pointer to either, whose nil is sealed as a +// null as long as a value or stored as SQL NULL — and seals as vitaminc's +// tagged leaf, which a // Rust record opens when its field is a Value (not a bare u32 or String). A struct, slice or map seals only as part of an opaque struct, // which crosses as one JSON document and is one leaf; stashgen refuses it // anywhere else, because the engine would seal it as a tree of leaves and a diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index fa94c2a16..657458264 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -154,6 +154,10 @@ func (c *Codec[P, E]) Decrypt(ctx context.Context, d encrypt.Decrypter, encrypte switch { case f.verb == verbOmit: continue + case !ok && f.null == NullSQL: + // The field was nil, so nothing was stored for it; it opens + // as nil. + continue case !ok: return nil, fmt.Errorf("gensupport: %s: value %d: Open gave no field %q", c.g.TypeName, i, f.name) case f.verb == verbContextField: @@ -190,10 +194,13 @@ func (c *Codec[P, E]) Decrypt(ctx context.Context, d encrypt.Decrypter, encrypte } // split sorts a value's fields into what crosses the binding and what stays: -// every declared field must be present and nothing else may be. +// every declared field must be present and nothing else may be. A nil field +// declared with NullSQL is left out of the row, since nothing is stored for +// it; one declared with NullSealed crosses as nil, which the engine seals. func (c *Codec[P, E]) split(vals Values) (record.Source, map[string]any, error) { row := make(record.Source, len(c.plan.Fields)) keep := map[string]any{} + left := 0 for _, f := range c.g.Declaration.fields { if f.verb == verbOmit { continue @@ -202,6 +209,10 @@ func (c *Codec[P, E]) split(vals Values) (record.Source, map[string]any, error) if !ok { return nil, nil, fmt.Errorf("the generated Source gave no field %q", f.name) } + if v == nil && f.null == NullSQL { + left++ + continue + } if f.crosses() { converted, err := asKind(f.kind, v) if err != nil { @@ -212,7 +223,7 @@ func (c *Codec[P, E]) split(vals Values) (record.Source, map[string]any, error) keep[f.name] = v } } - if len(vals) != len(row)+len(keep) { + if len(vals) != len(row)+len(keep)+left { for name := range vals { if _, ok := row[name]; ok { continue @@ -297,6 +308,33 @@ func Get[T any](vals Values, name string) (T, error) { return out, nil } +// Deref is a field declared over a pointer as Source gives it: nil for a nil +// pointer, else the value it points to. +func Deref[T any](p *T) any { + if p == nil { + return nil + } + return *p +} + +// GetPtr reads one opened field declared over a pointer: nil for a field +// that opened as nil, which it stored as a null or as nothing, else a +// pointer to the value as [Get] reads it. +func GetPtr[T any](vals Values, name string) (*T, error) { + v, ok := vals[name] + if !ok { + return nil, fmt.Errorf("gensupport: the opened value has no field %q", name) + } + if v == nil { + return nil, nil + } + out, err := Get[T](vals, name) + if err != nil { + return nil, err + } + return &out, nil +} + // Records wraps a codec with a model's conversions, for separate columns: // EncryptRows and DecryptRows return and take the model. func Records[P, E, R any](codec *Codec[P, E], to func(E) R, from func(R) E) *RecordsCodec[P, R] { diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index 57d3e8c31..fc090762e 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -76,8 +76,24 @@ type field struct { verb verb indexes []encrypt.Index eqlType string + null NullMode } +// NullMode is how a sealed field declared over a pointer (*T) stores nil: +// stashgen's `null=` tag. A field with none is never nil. +type NullMode string + +const ( + // NullSealed seals nil like a value, as a null as long as a value of + // the field's kind, so the stored record does not show which rows are + // absent. Decrypt gives nil. The field derives no index, and its kind + // has a fixed width. + NullSealed NullMode = "sealed" + // NullSQL stores nothing for nil: the field's columns hold SQL NULL, + // which shows that the value is absent. Decrypt gives nil. + NullSQL NullMode = "sql" +) + // Declaration is a generated struct's declaration as data: its context and, // for each field, what happens to it. Only generated code builds one, from // the struct's stash tags, and only generated code reads it. A mistake in a @@ -152,6 +168,34 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara return d.add(field{name: name, kind: kind, verb: verbEncryptInto, eqlType: eqlType}) } +// Null says how the named field, declared over a pointer, stores nil. The +// engine decides whether the field can, when the declaration is first used: +// a sealed null needs a field with no index, of a kind of fixed width. +func (d Declaration) Null(name string, mode NullMode) Declaration { + if d.err != nil { + return d + } + if mode != NullSealed && mode != NullSQL { + d.err = fmt.Errorf("gensupport: Null(%q): unknown mode %q", name, mode) + return d + } + for i := range d.fields { + if d.fields[i].name != name { + continue + } + if !d.fields[i].sealed() { + d.err = fmt.Errorf("gensupport: Null(%q): only a sealed field stores nil", name) + return d + } + // A copy, so the receiver's fields are not changed under it. + d.fields = slices.Clone(d.fields) + d.fields[i].null = mode + return d + } + d.err = fmt.Errorf("gensupport: Null(%q): no such field", name) + return d +} + // Omit leaves the field out: it does not cross the binding and is not // stored. The declaration lists it so a reader sees the choice. func (d Declaration) Omit(name string) Declaration { @@ -238,7 +282,7 @@ func (d Declaration) plan() (*record.Plan, error) { if !f.sealed() { continue } - rf := record.Field{Name: f.name, Identity: f.identity, Kind: record.Kind(f.kind)} + rf := record.Field{Name: f.name, Identity: f.identity, Kind: record.Kind(f.kind), Null: record.Null(f.null)} switch f.verb { case verbEncryptInto: // The EQL type's own plan decides the outputs; the engine diff --git a/languages/golang/encrypt/internal/testusers/birthday_stash.go b/languages/golang/encrypt/internal/testusers/birthday_stash.go new file mode 100644 index 000000000..78e3e2206 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/birthday_stash.go @@ -0,0 +1,94 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Birthday prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedBirthday struct { + DOB eql.Date +} + +func (e EncryptedBirthday) String() string { + return gensupport.Redacted("EncryptedBirthday", nil, "DOB") +} + +func (e EncryptedBirthday) LogValue() slog.Value { + return gensupport.RedactedLog(nil, "DOB") +} + +// Stops compiling when Birthday gains, loses, reorders or retypes a field. +var _ = birthdayShape(Birthday{}) + +type birthdayShape struct { + _ struct{} + DOB encrypt.Date +} + +var birthdayDeclaration = gensupport.Declare("people"). + EncryptInto("dob", gensupport.Date, "Date") + +var birthdayCodec = gensupport.New(gensupport.Generated[Birthday, EncryptedBirthday]{ + TypeName: "Birthday", + Declaration: birthdayDeclaration, + PrintsPlaintext: true, + Source: func(v Birthday) gensupport.Values { + return gensupport.Values{"dob": v.DOB} + }, + Seal: func(rec gensupport.Record) (EncryptedBirthday, error) { + var e EncryptedBirthday + e.DOB = eql.Date(rec["dob"].EQL) + return e, nil + }, + Open: func(e EncryptedBirthday) gensupport.Record { + return gensupport.Record{"dob": {EQL: e.DOB}} + }, + Value: func(e EncryptedBirthday, vals gensupport.Values) (Birthday, error) { + var v Birthday + var err error + if v.DOB, err = gensupport.Get[encrypt.Date](vals, "dob"); err != nil { + return Birthday{}, err + } + return v, nil + }, +}) + +// EncryptBirthday seals each Birthday, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. The result has one +// element for each input, in the same order. +func EncryptBirthday(ctx context.Context, cipher *encrypt.Cipher, values []Birthday) ([]EncryptedBirthday, error) { + return birthdayCodec.Encrypt(ctx, cipher, values) +} + +// DecryptBirthday opens each EncryptedBirthday, with one ZeroKMS request for +// each 500 sealed values, plus one the first time a keyset is used. +func DecryptBirthday(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedBirthday) ([]Birthday, error) { + return birthdayCodec.Decrypt(ctx, d, encrypted) +} + +var BirthdayFields = struct { + DOB BirthdayDOBField +}{ + DOB: BirthdayDOBField{gensupport.NewField[encrypt.Date](birthdayDeclaration, "dob")}, +} + +type BirthdayDOBField struct { + field gensupport.Field[encrypt.Date] +} + +func (f BirthdayDOBField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.Date, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Date(out.EQL), err +} diff --git a/languages/golang/encrypt/internal/testusers/people.go b/languages/golang/encrypt/internal/testusers/people.go new file mode 100644 index 000000000..6ead84862 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/people.go @@ -0,0 +1,28 @@ +package testusers + +import "github.com/cipherstash/stack/languages/golang/encrypt" + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Person -name Person +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Birthday -name Birthday + +// Person has optional fields, each declared over a pointer, in both ways a +// field stores nil (#1298): DOB and Age as a sealed null, as long as a +// value, which the generator picks for a type of fixed width with no index; +// Born, Score and Nick as SQL NULL, which an indexed field and text have to +// ask for. +type Person struct { + _ struct{} `stash:"context=people"` + ID int64 `stash:"id,passthrough"` + DOB *encrypt.Date `stash:"dob,encrypt_into=Date"` + Born *encrypt.Date `stash:"born,encrypt_into=DateEq,null=sql"` + Age *int32 `stash:"age,encrypt"` + Score *int64 `stash:"score,encrypt,index=ore,null=sql"` + Nick *string `stash:"nick,encrypt,null=sql"` +} + +// Birthday reads Person's dob column into a field that is not a pointer, +// so a stored null has nowhere to go. +type Birthday struct { + _ struct{} `stash:"context=people"` + DOB encrypt.Date `stash:"dob,encrypt_into=Date"` +} diff --git a/languages/golang/encrypt/internal/testusers/person_stash.go b/languages/golang/encrypt/internal/testusers/person_stash.go new file mode 100644 index 000000000..c135ea508 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/person_stash.go @@ -0,0 +1,260 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Person prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedPerson struct { + ID int64 + DOB eql.Date + Born *eql.DateEq + Age EncryptedPersonAge + Score *EncryptedPersonScore + Nick *EncryptedPersonNick +} + +type EncryptedPersonAge struct { + Ciphertext encrypt.Ciphertext +} + +type EncryptedPersonScore struct { + Ciphertext encrypt.Ciphertext + Ore encrypt.OreTerm +} + +type EncryptedPersonNick struct { + Ciphertext encrypt.Ciphertext +} + +func (e EncryptedPerson) String() string { + return gensupport.Redacted("EncryptedPerson", map[string]any{"ID": e.ID}, "DOB", "Born", "Age", "Score", "Nick") +} + +func (e EncryptedPerson) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "DOB", "Born", "Age", "Score", "Nick") +} + +// Stops compiling when Person gains, loses, reorders or retypes a field. +var _ = personShape(Person{}) + +type personShape struct { + _ struct{} + ID int64 + DOB *encrypt.Date + Born *encrypt.Date + Age *int32 + Score *int64 + Nick *string +} + +var personDeclaration = gensupport.Declare("people"). + Passthrough("id"). + EncryptInto("dob", gensupport.Date, "Date"). + Null("dob", gensupport.NullSealed). + EncryptInto("born", gensupport.Date, "DateEq"). + Null("born", gensupport.NullSQL). + Encrypt("age", gensupport.Int32). + Null("age", gensupport.NullSealed). + EncryptIndex("score", gensupport.Int64, encrypt.Ore). + Null("score", gensupport.NullSQL). + Encrypt("nick", gensupport.String). + Null("nick", gensupport.NullSQL) + +var personCodec = gensupport.New(gensupport.Generated[Person, EncryptedPerson]{ + TypeName: "Person", + Declaration: personDeclaration, + PrintsPlaintext: true, + Source: func(v Person) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "dob": gensupport.Deref(v.DOB), + "born": gensupport.Deref(v.Born), + "age": gensupport.Deref(v.Age), + "score": gensupport.Deref(v.Score), + "nick": gensupport.Deref(v.Nick), + } + }, + Seal: func(rec gensupport.Record) (EncryptedPerson, error) { + var e EncryptedPerson + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedPerson{}, err + } + e.DOB = eql.Date(rec["dob"].EQL) + if o, ok := rec["born"]; ok { + v := eql.DateEq(o.EQL) + e.Born = &v + } + e.Age = EncryptedPersonAge{Ciphertext: rec["age"].Ciphertext} + if o, ok := rec["score"]; ok { + v := EncryptedPersonScore{ + Ciphertext: o.Ciphertext, + Ore: o.Ore, + } + e.Score = &v + } + if o, ok := rec["nick"]; ok { + v := EncryptedPersonNick{Ciphertext: o.Ciphertext} + e.Nick = &v + } + return e, nil + }, + Open: func(e EncryptedPerson) gensupport.Record { + rec := gensupport.Record{ + "id": {Value: e.ID}, + "dob": {EQL: e.DOB}, + "age": {Ciphertext: e.Age.Ciphertext}, + } + if e.Born != nil { + rec["born"] = gensupport.Output{EQL: *e.Born} + } + if e.Score != nil { + rec["score"] = gensupport.Output{Ciphertext: e.Score.Ciphertext, Ore: e.Score.Ore} + } + if e.Nick != nil { + rec["nick"] = gensupport.Output{Ciphertext: e.Nick.Ciphertext} + } + return rec + }, + Value: func(e EncryptedPerson, vals gensupport.Values) (Person, error) { + var v Person + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Person{}, err + } + if v.DOB, err = gensupport.GetPtr[encrypt.Date](vals, "dob"); err != nil { + return Person{}, err + } + if v.Born, err = gensupport.GetPtr[encrypt.Date](vals, "born"); err != nil { + return Person{}, err + } + if v.Age, err = gensupport.GetPtr[int32](vals, "age"); err != nil { + return Person{}, err + } + if v.Score, err = gensupport.GetPtr[int64](vals, "score"); err != nil { + return Person{}, err + } + if v.Nick, err = gensupport.GetPtr[string](vals, "nick"); err != nil { + return Person{}, err + } + return v, nil + }, +}) + +// EncryptPerson seals each Person, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func EncryptPerson(ctx context.Context, cipher *encrypt.Cipher, values []Person) ([]EncryptedPerson, error) { + return personCodec.Encrypt(ctx, cipher, values) +} + +// DecryptPerson opens each EncryptedPerson, with one ZeroKMS request for each +// 500 sealed values, plus one the first time a keyset is used. +func DecryptPerson(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedPerson) ([]Person, error) { + return personCodec.Decrypt(ctx, d, encrypted) +} + +var PersonFields = struct { + DOB PersonDOBField + Born PersonBornField + Age PersonAgeField + Score PersonScoreField + Nick PersonNickField +}{ + DOB: PersonDOBField{gensupport.NewField[encrypt.Date](personDeclaration, "dob")}, + Born: PersonBornField{gensupport.NewField[encrypt.Date](personDeclaration, "born")}, + Age: PersonAgeField{gensupport.NewField[int32](personDeclaration, "age")}, + Score: PersonScoreField{gensupport.NewField[int64](personDeclaration, "score")}, + Nick: PersonNickField{gensupport.NewField[string](personDeclaration, "nick")}, +} + +type PersonDOBField struct { + field gensupport.Field[encrypt.Date] +} + +func (f PersonDOBField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.Date, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Date(out.EQL), err +} + +type PersonBornField struct { + field gensupport.Field[encrypt.Date] +} + +func (f PersonBornField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.DateEq, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.DateEq(out.EQL), err +} + +func (f PersonBornField) Eq(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.DateEqQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.DateEqQuery(term), err +} + +type PersonAgeField struct { + field gensupport.Field[int32] +} + +func (f PersonAgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) (EncryptedPersonAge, error) { + out, err := f.field.Encrypt(ctx, c, v) + return EncryptedPersonAge{Ciphertext: out.Ciphertext}, err +} + +type PersonScoreField struct { + field gensupport.Field[int64] +} + +func (f PersonScoreField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int64) (EncryptedPersonScore, error) { + out, err := f.field.Encrypt(ctx, c, v) + if err != nil { + return EncryptedPersonScore{}, err + } + return EncryptedPersonScore{Ciphertext: out.Ciphertext, Ore: out.Ore}, nil +} + +func (f PersonScoreField) Lt(ctx context.Context, c *encrypt.Cipher, v int64) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "lt", v) + return encrypt.OreTerm(term), err +} + +func (f PersonScoreField) Lte(ctx context.Context, c *encrypt.Cipher, v int64) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "lte", v) + return encrypt.OreTerm(term), err +} + +func (f PersonScoreField) Gt(ctx context.Context, c *encrypt.Cipher, v int64) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "gt", v) + return encrypt.OreTerm(term), err +} + +func (f PersonScoreField) Gte(ctx context.Context, c *encrypt.Cipher, v int64) (encrypt.OreTerm, error) { + term, err := f.field.Term(ctx, c, "gte", v) + return encrypt.OreTerm(term), err +} + +func (f PersonScoreField) Between(ctx context.Context, c *encrypt.Cipher, lo, hi int64) (encrypt.Range[encrypt.OreTerm], error) { + low, high, err := f.field.Range(ctx, c, "between", lo, hi) + return encrypt.Range[encrypt.OreTerm]{Low: low, High: high}, err +} + +type PersonNickField struct { + field gensupport.Field[string] +} + +func (f PersonNickField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (EncryptedPersonNick, error) { + out, err := f.field.Encrypt(ctx, c, v) + return EncryptedPersonNick{Ciphertext: out.Ciphertext}, err +} diff --git a/languages/golang/encrypt/null_test.go b/languages/golang/encrypt/null_test.go new file mode 100644 index 000000000..997745819 --- /dev/null +++ b/languages/golang/encrypt/null_test.go @@ -0,0 +1,93 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Optional fields, declared over pointers (#1298), through generated code +// over the deterministic guest build with the EQL types. A sealed null is as +// long as a value and the stored field never shows it; a SQL null stores +// nothing. Both decrypt to nil. + +func ptr[T any](v T) *T { return &v } + +// cOf is the length of the `c` an EQL value stores. +func cOf(t *testing.T, value []byte) int { + t.Helper() + var envelope struct { + C string `json:"c"` + } + if err := json.Unmarshal(value, &envelope); err != nil { + t.Fatalf("the EQL value is not JSON: %v", err) + } + if envelope.C == "" { + t.Fatalf("the EQL value has no c: %s", value) + } + return len(envelope.C) +} + +func TestOptionalFieldsStoreNilAsTheirTagsSay(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + dob, born := encrypt.Date{Year: 1990, Month: 5, Day: 17}, encrypt.Date{Year: 1991, Month: 6, Day: 1} + people := []testusers.Person{ + {ID: 1, DOB: &dob, Born: &born, Age: ptr(int32(34)), Score: ptr(int64(7)), Nick: ptr("al")}, + {ID: 2}, + } + sealed, err := testusers.EncryptPerson(ctx, cipher, people) + if err != nil { + t.Fatal(err) + } + present, absent := sealed[0], sealed[1] + + // A sealed null: a value whose ciphertext is as long as a date's, or + // an int32's, so the column does not show which rows are absent. + if len(absent.DOB) == 0 || cOf(t, absent.DOB) != cOf(t, present.DOB) { + t.Fatalf("a nil DOB is sealed as long as a date: %s, beside %s", absent.DOB, present.DOB) + } + if len(absent.Age.Ciphertext) != len(present.Age.Ciphertext) { + t.Fatalf("a nil Age is sealed as long as an int32: %d bytes, beside %d", len(absent.Age.Ciphertext), len(present.Age.Ciphertext)) + } + // A SQL null: nothing stored, so the column holds NULL. + if absent.Born != nil || absent.Score != nil || absent.Nick != nil { + t.Fatalf("a nil null=sql field stores nothing: %+v", absent) + } + if present.Born == nil || present.Score == nil || present.Nick == nil { + t.Fatalf("a present null=sql field is stored: %+v", present) + } + var stored eql.Date + if err := stored.Scan([]byte(absent.DOB)); err != nil { + t.Fatalf("a sealed null is a value the column scans: %v", err) + } + + opened, err := testusers.DecryptPerson(ctx, cipher, sealed) + if err != nil { + t.Fatal(err) + } + if got := opened[1]; got.ID != 2 || got.DOB != nil || got.Born != nil || got.Age != nil || got.Score != nil || got.Nick != nil { + t.Fatalf("every nil decrypts to nil: %+v", got) + } + got := opened[0] + if got.DOB == nil || *got.DOB != dob || got.Born == nil || *got.Born != born || got.Age == nil || *got.Age != 34 || + got.Score == nil || *got.Score != 7 || got.Nick == nil || *got.Nick != "al" { + t.Fatalf("every value decrypts as it was: %+v", got) + } + + // A field that is not a pointer has no nil to give: the stored null is + // a decrypt error, not a zero date. + _, err = testusers.DecryptBirthday(ctx, cipher, []testusers.EncryptedBirthday{{DOB: absent.DOB}}) + if err == nil { + t.Fatal("a sealed null decrypted into encrypt.Date, which has no nil") + } + birthdays, err := testusers.DecryptBirthday(ctx, cipher, []testusers.EncryptedBirthday{{DOB: present.DOB}}) + if err != nil || birthdays[0].DOB != dob { + t.Fatalf("a present date reads into either: %v, %+v", err, birthdays) + } +} diff --git a/languages/golang/encrypt/records.go b/languages/golang/encrypt/records.go index 6ff270867..657b7e786 100644 --- a/languages/golang/encrypt/records.go +++ b/languages/golang/encrypt/records.go @@ -252,6 +252,10 @@ func sourceRow(p *record.Plan, row record.Source) (vcvalue.Object, error) { } for _, f := range p.Fields { v, ok := row[f.Name] + if f.Null == record.SQLNull && (!ok || v == nil) { + // Nothing is stored for it: the engine leaves it out. + continue + } if !ok { return nil, fmt.Errorf("no value for field %q", f.Name) } @@ -323,7 +327,8 @@ func sealedOf(p *record.Plan, node any) (record.Sealed, error) { sealed[name] = o } for _, f := range p.Fields { - if _, ok := sealed[f.Name]; !ok { + // A SQLNull field that was nil has no entry: nothing was stored. + if _, ok := sealed[f.Name]; !ok && f.Null != record.SQLNull { return nil, fmt.Errorf("the engine returned no outputs for field %q", f.Name) } } @@ -379,6 +384,11 @@ func (c *Client) open(ctx context.Context, sel KeysetSelector, expected string, continue } outputs, ok := rec[f.Name] + if !ok && f.Null == record.SQLNull { + // It was nil, and nothing was stored: the engine opens it + // as nil. + continue + } if f.IsTarget() { if !ok || outputs.EQL == nil { return nil, fmt.Errorf("%w: row %d, field %q", errNoEQL, i, f.Name) diff --git a/languages/golang/encrypt/values.go b/languages/golang/encrypt/values.go index 140c52178..caf4a854a 100644 --- a/languages/golang/encrypt/values.go +++ b/languages/golang/encrypt/values.go @@ -23,9 +23,9 @@ import ( // // The zero Date (year 0, month 0, day 0) is not a date: a Date field must be // set before it is encrypted or queried, and a zero one is refused with -// ErrEncoding, naming the field. A sealed field cannot be a pointer, so an -// optional date has no sealed form of its own yet: keep it in an opaque -// struct, or store a sentinel date your program treats as absent. +// ErrEncoding, naming the field. An optional date is a *Date field: nil is +// stored as a sealed null as long as a date, or, with the tag's null=sql, as +// SQL NULL (see stashgen's README, "Optional fields"). type Date = vcvalue.Date // DateOf is the calendar day t falls on in its own location: what t.Date() diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 448dfdbfc..c2c8db24e 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -106,8 +106,30 @@ type Field struct { // type's own plan decides what is sealed and which terms sit beside it, // and the engine returns the finished value under EQLKey. Target string + // Null is how the field stores a value that is not there: NotNull for a + // field every row has, SealedNull or SQLNull for one declared over a + // pointer. + Null Null } +// Null is how a field stores a value that is not there: the plan's "null". +// The spellings are wire format, stack-encrypt's dynamic::NullMode. +type Null string + +const ( + // NotNull is a field every row has. A typed field refuses a nil. + NotNull Null = "" + // SealedNull seals a nil like a value, as a typed null as long as one of + // the field's kind, so the stored record does not show which rows are + // absent; it opens as nil. The engine takes it on a field that seals and + // derives no term, of a kind of fixed width. + SealedNull Null = "sealed" + // SQLNull stores nothing for a nil: the field is left out of the record, + // a source may leave it out, and a stored record without it opens it as + // nil. Its columns hold SQL NULL, which shows that the value is absent. + SQLNull Null = "sql" +) + // IsTarget reports whether the field names an EQL type. func (f Field) IsTarget() bool { return f.Target != "" } @@ -261,6 +283,11 @@ func (p *Plan) Validate() error { if !f.Kind.Known() { return fmt.Errorf("record: field %q: unknown kind %q", f.Name, f.Kind) } + switch f.Null { + case NotNull, SealedNull, SQLNull: + default: + return fmt.Errorf("record: field %q: unknown null %q", f.Name, f.Null) + } if f.IsTarget() { if len(f.Outputs) != 0 { return fmt.Errorf("record: field %q names the EQL type %s and outputs; a field has one or the other", f.Name, f.Target) @@ -358,6 +385,9 @@ func (p *Plan) Wire() vcvalue.Object { if f.Kind != Untyped { spec = append(spec, vcvalue.Field{Key: "type", Value: string(f.Kind)}) } + if f.Null != NotNull { + spec = append(spec, vcvalue.Field{Key: "null", Value: string(f.Null)}) + } out = append(out, vcvalue.Field{Key: f.Name, Value: spec}) } return out @@ -404,7 +434,8 @@ func (p *Plan) Descriptor(f Field) string { // Source is one record's plaintext for the engine: each sealed field's value // by name, and the context field's string when the plan has one. -// Passthrough and omitted fields are not in it. +// Passthrough and omitted fields are not in it, nor need a SQLNull field +// that is nil be. type Source = map[string]any // Outputs is what the engine produced for one sealed field, or what it @@ -423,7 +454,8 @@ type Outputs struct { } // Sealed is one record as stored: each sealed field's outputs by name, and -// the context field's under its name when the plan has one. +// the context field's under its name when the plan has one. A SQLNull field +// that was nil has no entry. type Sealed = map[string]Outputs // Target is one EQL type as se_targets lists it. The entry's keys are wire diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index 273a7c978..7543f6305 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -47,6 +47,39 @@ type Field struct { // a column that was renamed keeps the identity it was first written // under. "" means Name. Only a policy sets it. Identity string + // Null is how a sealed field declared over a pointer stores nil; NotNull + // for every other field. GoType is then the type the pointer points to. + Null Null +} + +// Null is how a sealed field declared over a pointer (*T) stores nil: the +// `null=` tag, or the generator's choice where the tag is left out. +type Null string + +const ( + // NotNull is a field that is not a pointer: it is never nil. + NotNull Null = "" + // NullSealed seals nil like a value, as long as one, so the stored + // field does not show that the value is absent. The default for a field + // with no index whose type has a fixed width: a bool, a number, + // encrypt.Date, time.Time, encrypt.Decimal, encrypt.Int128 or + // encrypt.Uint128. + NullSealed Null = "sealed" + // NullSQL stores nothing for nil: the field's columns hold SQL NULL, + // which shows that the value is absent. A field with an index, or of + // text or bytes, says so with `null=sql`: its absence would show anyway. + NullSQL Null = "sql" +) + +// fixedWidth reports whether a scalar kind seals at one length whatever its +// value, so that a sealed nil can be as long as a value. +func (k Kind) fixedWidth() bool { + switch k { + case KindBool, KindInt, KindUint, KindFloat, KindInt128, KindUint128, + KindDate, KindTimestamp, KindDecimal: + return true + } + return false } // Verb is what happens to a field. diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 1ac6e7256..2cb8af7ad 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -294,7 +294,7 @@ func (w *writer) declaration(f *genFile) { } for i, fld := range fields { end := "." - if i == len(fields)-1 && fld.Identity == "" { + if i == len(fields)-1 && fld.Identity == "" && fld.Null == NotNull { end = "" } switch fld.Verb { @@ -323,12 +323,27 @@ func (w *writer) declaration(f *genFile) { } if fld.Identity != "" { end = "." - if i == len(fields)-1 { + if i == len(fields)-1 && fld.Null == NotNull { end = "" } w.p("\tIdentity(%q, %q)%s", fld.Name, fld.Identity, end) } + if fld.Null != NotNull { + end = "." + if i == len(fields)-1 { + end = "" + } + w.p("\tNull(%q, %s)%s", fld.Name, nullExpr(fld.Null), end) + } + } +} + +// nullExpr is a null mode's constant in gensupport. +func nullExpr(n Null) string { + if n == NullSQL { + return "gensupport.NullSQL" } + return "gensupport.NullSealed" } // literal writes a composite literal: one line with one entry, one entry per @@ -384,6 +399,11 @@ func (w *writer) source(f *genFile) { } else { entries := make([]string, len(f.fields)) for i, g := range f.fields { + if g.nullable() { + // nil crosses as nil, not as a nil pointer. + entries[i] = fmt.Sprintf("%q: gensupport.Deref(v.%s)", g.Name, g.GoName) + continue + } entries[i] = fmt.Sprintf("%q: v.%s", g.Name, g.GoName) } w.literal("\t\t", "return gensupport.Values{", entries, "}") @@ -419,17 +439,35 @@ func (w *writer) seal(f *genFile) { w.p("\t\t}") } for _, g := range f.fields { - switch g.Verb { - case VerbPassthrough, VerbContextField: - case VerbEncryptInto: - w.p("\t\te.%s = %s(rec[%q].EQL)", g.GoName, g.outputType, g.Name) - default: - entries := make([]string, len(g.outputs)) - for i, o := range g.outputs { - entries[i] = fmt.Sprintf("%s: rec[%q].%s", o.name, g.Name, o.name) + if g.Stored() { + continue + } + if g.Null == NullSQL { + // A nil stores nothing: the record has no entry, and the + // member stays nil, so its columns hold SQL NULL. + w.p("\t\tif o, ok := rec[%q]; ok {", g.Name) + if g.Verb == VerbEncryptInto { + w.p("\t\t\tv := %s(o.EQL)", g.outputType) + } else { + entries := make([]string, len(g.outputs)) + for i, o := range g.outputs { + entries[i] = fmt.Sprintf("%s: o.%s", o.name, o.name) + } + w.literal("\t\t\t", fmt.Sprintf("v := %s{", g.outputType), entries, "}") } - w.literal("\t\t", fmt.Sprintf("e.%s = %s{", g.GoName, g.outputType), entries, "}") + w.p("\t\t\te.%s = &v", g.GoName) + w.p("\t\t}") + continue + } + if g.Verb == VerbEncryptInto { + w.p("\t\te.%s = %s(rec[%q].EQL)", g.GoName, g.outputType, g.Name) + continue } + entries := make([]string, len(g.outputs)) + for i, o := range g.outputs { + entries[i] = fmt.Sprintf("%s: rec[%q].%s", o.name, g.Name, o.name) + } + w.literal("\t\t", fmt.Sprintf("e.%s = %s{", g.GoName, g.outputType), entries, "}") } w.p("\t\treturn e, nil") w.p("\t},") @@ -442,22 +480,48 @@ func (w *writer) open(f *genFile) { w.p("\t},") return } - entries := make([]string, len(f.fields)) - for i, g := range f.fields { + output := func(g genField, e string) string { switch g.Verb { case VerbPassthrough, VerbContextField: - entries[i] = fmt.Sprintf("%q: {Value: e.%s}", g.Name, g.GoName) + return fmt.Sprintf("{Value: %s}", e) case VerbEncryptInto: - entries[i] = fmt.Sprintf("%q: {EQL: e.%s}", g.Name, g.GoName) - default: - parts := make([]string, len(g.outputs)) - for n, o := range g.outputs { - parts[n] = fmt.Sprintf("%s: e.%s.%s", o.name, g.GoName, o.name) - } - entries[i] = fmt.Sprintf("%q: {%s}", g.Name, strings.Join(parts, ", ")) + return fmt.Sprintf("{EQL: %s}", e) + } + parts := make([]string, len(g.outputs)) + for n, o := range g.outputs { + parts[n] = fmt.Sprintf("%s: %s.%s", o.name, e, o.name) + } + return "{" + strings.Join(parts, ", ") + "}" + } + var entries []string + var absent []genField + for _, g := range f.fields { + if g.Null == NullSQL { + absent = append(absent, g) + continue } + entries = append(entries, fmt.Sprintf("%q: %s", g.Name, output(g, "e."+g.GoName))) } - w.literal("\t\t", "return gensupport.Record{", entries, "}") + if len(absent) == 0 { + w.literal("\t\t", "return gensupport.Record{", entries, "}") + w.p("\t},") + return + } + w.literal("\t\t", "rec := gensupport.Record{", entries, "}") + for _, g := range absent { + // A nil member stored nothing: the record leaves it out, and it + // opens as nil. + w.p("\t\tif e.%s != nil {", g.GoName) + // An EQL value is a slice and is dereferenced; an output struct's + // fields are read through the pointer. + member := "e." + g.GoName + if g.Verb == VerbEncryptInto { + member = "*" + member + } + w.p("\t\t\trec[%q] = gensupport.Output%s", g.Name, output(g, member)) + w.p("\t\t}") + } + w.p("\t\treturn rec") w.p("\t},") } @@ -491,6 +555,25 @@ func (w *writer) value(f *genFile) { // for it: the struct gets the field's zero value. continue } + if g.nullable() && g.definedScalar() { + // The engine returns the underlying type, or nil; the + // conversion to the defined type is the struct's own. + w.p("\t\traw%s, err := gensupport.GetPtr[%s](vals, %q)", g.GoName, g.GoType.Basic, g.Name) + w.p("\t\tif err != nil {") + fail() + w.p("\t\t}") + w.p("\t\tif raw%s != nil {", g.GoName) + w.p("\t\t\tv%s := %s(*raw%s)", g.GoName, g.valueExpr, g.GoName) + w.p("\t\t\tv.%s = &v%s", g.GoName, g.GoName) + w.p("\t\t}") + continue + } + if g.nullable() { + w.p("\t\tif v.%s, err = gensupport.GetPtr[%s](vals, %q); err != nil {", g.GoName, g.valueExpr, g.Name) + fail() + w.p("\t\t}") + continue + } if g.Sealed() && g.definedScalar() { // The engine returns the underlying type; the conversion // to the defined type is the struct's own. @@ -553,16 +636,16 @@ func (w *writer) fieldEntries(f *genFile) { } w.p("}{") for _, g := range sealed { - w.p("\t%s: %s{gensupport.NewField[%s](%s, %q)},", g.GoName, g.fieldType, g.typeExpr, f.declVar, g.Name) + w.p("\t%s: %s{gensupport.NewField[%s](%s, %q)},", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) } w.p("}") for _, g := range sealed { w.nl() w.p("type %s struct {", g.fieldType) - w.p("\tfield gensupport.Field[%s]", g.typeExpr) + w.p("\tfield gensupport.Field[%s]", g.valueExpr) w.p("}") w.nl() - w.p("func (f %s) Encrypt(ctx context.Context, c *encrypt.Cipher, v %s) (%s, error) {", g.fieldType, g.typeExpr, g.outputType) + w.p("func (f %s) Encrypt(ctx context.Context, c *encrypt.Cipher, v %s) (%s, error) {", g.fieldType, g.valueExpr, g.outputType) switch { case g.Verb == VerbEncryptInto: w.p("\tout, err := f.field.Encrypt(ctx, c, v)") @@ -669,7 +752,7 @@ func indexesAnswer(g *genField) ([]string, error) { } } } - notices = append(notices, fmt.Sprintf("stores %s terms that no generated method queries: the engine answers no query through the %s index on this field (%s)%s. Drop the index unless your own code compares the stored terms.", ix.Name, ix.Name, g.typeExpr, why)) + notices = append(notices, fmt.Sprintf("stores %s terms that no generated method queries: the engine answers no query through the %s index on this field (%s)%s. Drop the index unless your own code compares the stored terms.", ix.Name, ix.Name, g.valueExpr, why)) } return notices, nil } @@ -692,7 +775,7 @@ func queryFor(g *genField, op Operator) (queryMethod, error) { case op.Probabilistic: // The query value is the type's, and the query keeps what Verify // and TargetLimit need, as a MatchQuery does. - q.resultType = "encrypt.EQLMatchQuery[" + g.queryType + ", " + g.typeExpr + "]" + q.resultType = "encrypt.EQLMatchQuery[" + g.queryType + ", " + g.valueExpr + "]" q.termType = g.queryType return q, nil } @@ -702,7 +785,7 @@ func queryFor(g *genField, op Operator) (queryMethod, error) { termType := map[IndexName]string{IndexEquality: "encrypt.EqualityTerm", IndexOre: "encrypt.OreTerm", IndexOpe: "encrypt.OpeTerm"}[op.Index] switch { case op.Probabilistic && op.Index == IndexMatch && !op.Pair && g.GoType.Kind == KindString: - q.resultType = "encrypt.MatchQuery[" + g.typeExpr + "]" + q.resultType = "encrypt.MatchQuery[" + g.valueExpr + "]" case op.Probabilistic || termType == "": return q, fmt.Errorf("the engine answers %s through the %s index, and the SDK has no Go type for that query", op.Name, op.Index) case op.Pair && op.Index != IndexEquality: @@ -731,18 +814,18 @@ func (w *writer) query(g genField, q queryMethod) { switch { case q.operator.Probabilistic: w.b.WriteString(wrapComment(fmt.Sprintf("%s derives the %s query for v. Its answer can hold rows that do not match: decrypt the rows it fetches and keep those Verify reports true.", q.method, q.operator.Name), 80)) - w.p("%sv %s) (%s, error) {", recv, g.typeExpr, q.resultType) + w.p("%sv %s) (%s, error) {", recv, g.valueExpr, q.resultType) if g.Verb == VerbEncryptInto { w.p(" return gensupport.EQLMatches[%s](ctx, f.field, c, v)", q.termType) } else { w.p(" return gensupport.Matches(ctx, f.field, c, v)") } case q.operator.Pair: - w.p("%slo, hi %s) (%s, error) {", recv, g.typeExpr, q.resultType) + w.p("%slo, hi %s) (%s, error) {", recv, g.valueExpr, q.resultType) w.p(" low, high, err := f.field.Range(ctx, c, %q, lo, hi)", q.operator.Name) w.p(" return %s{Low: low, High: high}, err", q.resultType) default: - w.p("%sv %s) (%s, error) {", recv, g.typeExpr, q.resultType) + w.p("%sv %s) (%s, error) {", recv, g.valueExpr, q.resultType) w.p(" term, err := f.field.Term(ctx, c, %q, v)", q.operator.Name) w.p(" return %s(term), err", q.resultType) } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index 50255cf33..22294ffea 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -193,7 +193,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { if !f.Sealed() { continue } - rf := record.Field{Name: f.Name, Identity: f.Identity, Kind: wireKind(f.GoType)} + rf := record.Field{Name: f.Name, Identity: f.Identity, Kind: wireKind(f.GoType), Null: record.Null(f.Null)} switch f.Verb { case VerbEncryptInto: if len(eqlTypes) == 0 { diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index fd2b44ef6..f0a0e4765 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -93,6 +93,8 @@ type encMember struct { type genField struct { Field typeExpr string // the field's Go type + valueExpr string // the type of one value: typeExpr, or what a nullable field points to + memberType string // the encrypted type's member: outputType, or a pointer to it for NullSQL via string // the embedded passthrough struct it is promoted from, or "" tags string // the other libraries' tags, without stash outputType string // the encrypted type's field type: "eql.TextEq" or "EncryptedContactEmail" @@ -108,9 +110,13 @@ type genField struct { // scalar (type Email string): the engine returns the underlying type, and // the generated code converts. func (g genField) definedScalar() bool { - return g.GoType.Kind.Scalar() && g.GoType.Basic != "" && g.typeExpr != g.GoType.Basic + return g.GoType.Kind.Scalar() && g.GoType.Basic != "" && g.valueExpr != g.GoType.Basic } +// nullable reports whether the field is declared over a pointer, so its +// value may be nil. +func (g genField) nullable() bool { return g.Null != NotNull } + // output is one output of a sealed field in separate columns. type output struct { name string // "Ciphertext", "Equality" @@ -686,11 +692,34 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { field.Verb = VerbOmit } + // A sealed field declared over a pointer to one scalar may be nil: + // it is that scalar's field, and says how it stores nil (#1298). + pointer := field.Sealed() && field.GoType.Kind == KindPointer && field.GoType.Elem != nil && field.GoType.Elem.Kind.Scalar() + if pointer { + field.GoType = *field.GoType.Elem + } + if t.Null != NotNull && !pointer { + return fieldErr(typeName, cf.goName, "null= says how a field declared over a pointer stores nil, and a field of type %s is never nil; declare it as a pointer to one scalar, *%s", r.typeExpr(cf.typ), r.typeExpr(cf.typ)) + } + if pointer { + mode, err := r.nullMode(field, t.Null) + if err != nil { + return fieldErr(typeName, cf.goName, "%s", err) + } + field.Null = mode + } + if field.Null == NullSQL && len(r.req.Models) > 0 { + return fieldErr(typeName, cf.goName, "a null=sql field stores no columns when it is nil, and -model does not map such a field yet; leave out -model, or store nil as a sealed null") + } f.decl.Fields = append(f.decl.Fields, field) if t.Omit { continue } g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ)} + g.valueExpr = g.typeExpr + if pointer { + g.valueExpr = field.GoType.Name + } if g.Sealed() && !g.GoType.Kind.Scalar() { // The engine seals a composite as a tree of leaves, and a column // holds one leaf; only an opaque struct seals a whole value. @@ -744,6 +773,10 @@ func (r *reader) buildFields(c *collected) error { if g.Sealed() { g.fieldType = f.fieldTypePrefix + cf.goName + "Field" } + g.memberType = g.outputType + if g.Null == NullSQL { + g.memberType = "*" + g.outputType + } f.fields = append(f.fields, g) if cf.via != "" { if !embedded[cf.via] { @@ -752,11 +785,43 @@ func (r *reader) buildFields(c *collected) error { } continue } - f.members = append(f.members, encMember{name: cf.goName, typeExpr: g.outputType, tags: cf.tags}) + f.members = append(f.members, encMember{name: cf.goName, typeExpr: g.memberType, tags: cf.tags}) } return nil } +// nullMode is how a sealed field declared over a pointer stores nil: the +// tag's `null=` if it says, else NullSealed where a sealed nil hides that the +// value is absent. That needs a type of fixed width, so a nil is as long as a +// value, and no index, since a nil has no term yet (#1298) and its absence +// would show in the index column; anywhere else the field says null=sql, so +// a visible SQL NULL is a choice the source shows. +func (r *reader) nullMode(field Field, tagged Null) (Null, error) { + indexed := len(field.Indexes) > 0 + if field.Verb == VerbEncryptInto { + for _, eqlType := range r.eql { + if eqlType.Name == field.EQLType { + indexed = len(eqlType.Indexes) > 0 + } + } + } + fixed := field.GoType.Kind.fixedWidth() + switch { + case tagged == NullSQL: + return NullSQL, nil + case tagged == NullSealed && indexed: + return "", fmt.Errorf("a sealed nil has no index term yet, so the index column would show which values are absent; write null=sql, which stores SQL NULL") + case tagged == NullSealed && !fixed: + return "", fmt.Errorf("a sealed nil is as long as a value only for a type of fixed width, and a %s shows its length; write null=sql, which stores SQL NULL", field.GoType) + case fixed && !indexed: + return NullSealed, nil + case indexed: + return "", fmt.Errorf("a field with an index stores nil as SQL NULL, which shows which values are absent: a nil has no index term yet. Write null=sql to say so") + default: + return "", fmt.Errorf("a %s field stores nil as SQL NULL, which shows which values are absent: a %s shows its length, so a sealed nil would not hide it. Write null=sql to say so", "*"+field.GoType.Name, field.GoType) + } +} + func (r *reader) embeddedTags(fieldName string) string { _, st, err := lookupStruct(r.pkg, r.req.Type) if err != nil { diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 027e45a02..de22f8402 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -63,6 +63,18 @@ func TestRefusals(t *testing.T) { want string }{ {"an exported field with no stash tag", user(ctx + "\tEmail string"), stashgen.Request{Type: "User"}, "Email", "no stash tag"}, + // Optional fields (#1298): a pointer to one scalar, and how it stores nil. + {"a sealed pointer to a pointer", user(ctx + "\tAge **int32 `stash:\"age,encrypt\"`"), stashgen.Request{Type: "User"}, "Age", "a sealed field is one scalar"}, + {"a sealed pointer to a struct", user(ctx + "\tAddr *crm.Contact `stash:\"addr,encrypt\"`"), stashgen.Request{Type: "User"}, "Addr", "a sealed field is one scalar"}, + {"a sealed pointer to a slice", user(ctx + "\tTags *[]string `stash:\"tags,encrypt\"`"), stashgen.Request{Type: "User"}, "Tags", "a sealed field is one scalar"}, + {"null= on a field that is not a pointer", user(ctx + "\tAge int32 `stash:\"age,encrypt,null=sql\"`"), stashgen.Request{Type: "User"}, "Age", "a field of type int32 is never nil"}, + {"null= on a passthrough", user(ctx + "\tAge *int32 `stash:\"age,passthrough,null=sql\"`"), stashgen.Request{Type: "User"}, "Age", "keeps nil as nil"}, + {"an unknown null mode", user(ctx + "\tAge *int32 `stash:\"age,encrypt,null=zero\"`"), stashgen.Request{Type: "User"}, "Age", "null= is sealed or sql"}, + {"null= given twice", user(ctx + "\tAge *int32 `stash:\"age,encrypt,null=sql,null=sql\"`"), stashgen.Request{Type: "User"}, "Age", "null is given twice"}, + {"an indexed pointer that does not say null=sql", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore\"`"), stashgen.Request{Type: "User"}, "Age", "Write null=sql to say so"}, + {"a pointer to text that does not say null=sql", user(ctx + "\tNick *string `stash:\"nick,encrypt\"`"), stashgen.Request{Type: "User"}, "Nick", "a string shows its length"}, + {"a sealed null on an indexed pointer", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore,null=sealed\"`"), stashgen.Request{Type: "User"}, "Age", "a sealed nil has no index term yet"}, + {"a sealed null on a pointer to bytes", user(ctx + "\tBlob *[]byte `stash:\"blob,encrypt,null=sealed\"`"), stashgen.Request{Type: "User"}, "Blob", "a []byte shows its length"}, {"a match query on an index-only field", user(ctx + "\tName string `stash:\"name,index=match\"`"), stashgen.Request{Type: "User"}, "Name", "an index-only field stores no ciphertext to decrypt"}, {"a tag that does not parse", user(ctx + "\tEmail string `stash:\"email,shred\"`"), stashgen.Request{Type: "User"}, "Email", `unknown part "shred"`}, {"two fields with one name", user(ctx + "\tEmail string `stash:\"email,encrypt\"`\n\tAlt string `stash:\"email,encrypt\"`"), stashgen.Request{Type: "User"}, "Alt", `two fields write the name "email"`}, diff --git a/languages/golang/stashgen/tag.go b/languages/golang/stashgen/tag.go index 660577df8..655419ba4 100644 --- a/languages/golang/stashgen/tag.go +++ b/languages/golang/stashgen/tag.go @@ -22,6 +22,7 @@ const tagKey = "stash" // stash:"id,passthrough" Name, VerbPassthrough // stash:"tenant,context_field" Name, VerbContextField // stash:",passthrough" VerbPassthrough with no Name, on an embedded struct +// stash:"dob,encrypt_into=Date,null=sql" Null, on a sealed field declared over a pointer type tag struct { Omit bool Context string @@ -30,6 +31,9 @@ type tag struct { Verb Verb Indexes []Index EQLType string + // Null is how a sealed field declared over a pointer stores nil, when + // the tag says; the generator picks NullSealed where it can. + Null Null // Identity is set by a policy only; no tag spells it. Identity string } @@ -68,7 +72,7 @@ func parseTagValue(value string) (tag, error) { if strings.ContainsAny(t.Name, "=();") { return tag{}, fmt.Errorf("tag %q: the first part is the field's name, and %q is not a name", value, t.Name) } - var sawEncrypt, sawPassthrough, sawIndex, sawInto, sawContextField bool + var sawEncrypt, sawPassthrough, sawIndex, sawInto, sawContextField, sawNull bool for _, item := range items[1:] { switch { case item == "encrypt": @@ -104,15 +108,28 @@ func parseTagValue(value string) (tag, error) { if err != nil { return tag{}, fmt.Errorf("tag %q: %w", value, err) } + case strings.HasPrefix(item, "null="): + if sawNull { + return tag{}, fmt.Errorf("tag %q: null is given twice", value) + } + sawNull = true + switch mode := Null(strings.TrimPrefix(item, "null=")); mode { + case NullSealed, NullSQL: + t.Null = mode + default: + return tag{}, fmt.Errorf("tag %q: null= is sealed or sql, not %q", value, mode) + } case item == "opaque" || strings.HasPrefix(item, "context="): return tag{}, fmt.Errorf("tag %q: %s goes on the `_ struct{}` field, as `stash:\"context=...\"`", value, item) case item == "": return tag{}, fmt.Errorf("tag %q: an empty part", value) default: - return tag{}, fmt.Errorf("tag %q: unknown part %q; the parts are encrypt, encrypt_into=, index=, passthrough, context_field and -", value, item) + return tag{}, fmt.Errorf("tag %q: unknown part %q; the parts are encrypt, encrypt_into=, index=, null=, passthrough, context_field and -", value, item) } } switch { + case sawNull && (sawPassthrough || sawContextField): + return tag{}, fmt.Errorf("tag %q: null= says how a sealed field stores nil; a field stored as it is keeps nil as nil", value) case sawContextField && (sawEncrypt || sawInto || sawIndex || sawPassthrough): return tag{}, fmt.Errorf("tag %q: a context field is the context of the other fields, stored as it is; it takes no other part", value) case sawContextField: diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index dfb110020..eee7f4bf7 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -164,3 +164,25 @@ func NoticePrintsPlaintext(typeName string) {} // Identity gives a field a context part other than its name. func (d Declaration) Identity(name, identity string) Declaration { return d } + +// NullMode is how a sealed field declared over a pointer stores nil. +type NullMode string + +const ( + NullSealed NullMode = "sealed" + NullSQL NullMode = "sql" +) + +// Null says how the named field, declared over a pointer, stores nil. +func (d Declaration) Null(name string, mode NullMode) Declaration { return d } + +// Deref is a field declared over a pointer as Source gives it. +func Deref[T any](p *T) any { + if p == nil { + return nil + } + return *p +} + +// GetPtr reads one opened field declared over a pointer. +func GetPtr[T any](Values, string) (*T, error) { return nil, nil } diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 8d812938e..498bafbea 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -18,6 +18,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 fail only at a `ValueFrame: Decrypt` bound. The Go guest now takes vitaminc only this way. +- **A data plan field says how it stores a null** (`dynamic` feature, + #1298). A field spec takes `"null": "sealed"` or `"null": "sql"` + (`dynamic::NullMode`, `FieldPlan::with_null`). `"sealed"` seals a null + like a value, under the field's label, as vitaminc 0.8's typed null of + the field's type, as long as a present value, so the stored record does + not show which rows are absent; it opens as a null. It needs a declared + type of fixed width (a bool's plain null is already as long as its value) + and no term, on the field or on the EQL type it names: refused otherwise + with `Reason::SealedNullType` or `Reason::SealedNullIndex`. `"sql"` + stores nothing for a null, or for a row with no entry for the field: the + stored record has no entry, so a host writes SQL `NULL`, and a stored + record with no entry for the field opens it as a null. A batch's rows are + run by one lowered plan per set of fields they leave out, and every + group's requests go in the one ZeroKMS call. A passthrough takes no + `"null"` (`Reason::NullPassthrough`), and an unknown mode is + `Reason::UnknownNull`. A field with no `"null"` is unchanged: every row + has it, and a typed field refuses a null. - **A block ORE index, `OreBlock`** (ADR-0002 and ADR-0009, amended 2026-10-19). It derives ore-rs 0.9.0's 6-bit block ORE under its own key domain, over the same order keys as the CLLW indexes: From 06ce1de8f145d18206dcb68a9974e54a82159a82 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 20:06:05 +1100 Subject: [PATCH 04/14] test(eql): PostgreSQL stores a sealed null in a storage-only column The domain check of eql_v4_date takes a value whose c is a typed null, as it takes a date, and it reads back and opens as None (#1298). Runs in mise run test:encryption:postgres with the file's other database tests. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../encryption/tests/v4_date_time_numeric.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs index c451c4bf6..55d1f741f 100644 --- a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs +++ b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs @@ -987,3 +987,50 @@ fn refused_null( let null = ValueFrame::from_bytes(&[0x00]).unwrap(); refused(targets::encrypt(name, keyset, &label(), null)) } + +/// Needs an EQL v4 installation. A storage-only column stores an absent +/// value sealed as a typed null (#1298): the domain's check takes it, as +/// it takes a value, and it reads back and opens as `None`. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_stores_a_sealed_null_in_a_storage_only_column() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = runtime.block_on(common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = database(); + db.batch_execute( + "CREATE TEMP TABLE v4_optional (id int PRIMARY KEY, entry public.eql_v4_date);", + ) + .unwrap(); + let (absent, present): (date::Date, date::Date) = runtime.block_on(async { + ( + keyset.encrypt_as(&None::, column()).await.unwrap(), + keyset + .encrypt_as(&Date::from(day("2024-02-29")), column()) + .await + .unwrap(), + ) + }); + for (id, value) in [(1, &absent), (2, &present)] { + db.execute( + "INSERT INTO v4_optional (id, entry) VALUES ($1, $2::text::jsonb::public.eql_v4_date)", + &[&id, &json(value)], + ) + .unwrap(); + } + let stored: String = db + .query_one( + "SELECT entry::jsonb::text FROM v4_optional WHERE id = 1", + &[], + ) + .unwrap() + .get(0); + let stored: date::Date = serde_json::from_str(&stored).unwrap(); + let opened: Option = runtime + .block_on(async { cipher.decrypt_as(stored, column().into()).await }) + .unwrap(); + assert!(opened.is_none(), "the stored null opens as None"); +} From 0a0600dbdc2138f5fef5e9856bb630b51f3ff50e Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 20:25:44 +1100 Subject: [PATCH 05/14] test(stack-encrypt): a sealed null only in a termless target, and only through a nullable plan The two survivors of cargo mutants --in-diff on the null modes (#1298): no test refused "null": "sealed" on an EQL type with a term, and none read a stored null through a plan with no "null", where the field's kind check must refuse it rather than skip. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/record.rs | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index c3e05e6e3..5021ee115 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -5356,6 +5356,37 @@ mod tests { assert_eq!(open(&cipher, with, &plan).await, present); } + /// A stored null opens only through a plan that says the field may + /// hold one: read through a plan with no `"null"`, the field's kind + /// check refuses it, so a nil never reads as a zero value. + #[tokio::test] + async fn a_stored_null_is_refused_through_a_plan_that_has_no_null() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let sealing = plan(obj(vec![( + "dob", + nullable(label("dob"), &["c"], Some("date"), "sealed"), + )])) + .expect("parses"); + let reading = + plan(obj(vec![("dob", typed(label("dob"), &["c"], "date"))])).expect("parses"); + let sealed = seal(&keyset, obj(vec![("dob", Tree::Null)]), &sealing).await; + let refused = decrypt(Scope::Client(&cipher), sealed, &reading, None) + .expect("the record fits the plan") + .await + .err(); + assert!( + matches!(&refused, Some(crate::Error::Plan(crate::PlanError::FieldType { field, .. })) if field == "dob"), + "{refused:?}" + ); + // And through the plan that says so, a value opens as a value. + let sealed = seal(&keyset, obj(vec![("dob", date(19_000))]), &sealing).await; + assert_eq!( + open(&cipher, sealed, &sealing).await, + obj(vec![("dob", date(19_000))]) + ); + } + /// A null in a typed field with no `"null"` is still refused. #[tokio::test] async fn a_null_in_a_field_with_no_null_mode_is_refused() { @@ -6463,6 +6494,31 @@ mod tests { assert_eq!(generates(&cipher), 0, "nothing minted"); } + /// A target field stores a null as its plan says: under `"sql"` any + /// type, but sealed only in a type with no term, since a null has none + /// and the type's term column would show it (#1298). Refused when the + /// plan is built, naming the field, before any value crosses. + #[test] + fn a_target_field_seals_a_null_only_in_a_type_with_no_term() { + let with_null = |null: &str| { + let mut spec = target_spec(label("email"), TEXT_EQ); + if let Tree::Object(entries) = &mut spec { + entries.push(("null".to_string(), s(null))); + } + obj(vec![("email", spec)]) + }; + let refused = plan_with(with_null("sealed"), &FakeEql).err(); + assert!( + matches!( + &refused, + Some(Error::Plan { field: Some(f), reason: Reason::SealedNullIndex }) if f == "email" + ), + "{refused:?}" + ); + let parsed = plan_with(with_null("sql"), &FakeEql).expect("any type stores a SQL null"); + assert_eq!(parsed.fields()[0].null(), Some(NullMode::Sql)); + } + /// The resolver opens a target value through the client, which opens /// every keyset's values; `confine` is what holds it to the scope's /// keyset. A plan of one target field, so no lowered leaf can cause From b22be1111f5a41f7dd52fd6507d2219c6128411a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 21:02:37 +1100 Subject: [PATCH 06/14] feat: a null has an equality term, and a sealed null sits beside one vitaminc-normalize 0.8 derives an equality term for a null: one term per context, shared by every null, typed or plain. #1327 dropped it to keep the old behaviour; this keeps it (#1298). - stack-encrypt: `EqualityTerm` of a `None`, and `IndexSpec::Equality` on a null, now derive that term. No other index has one. - Data plans: `"null": "sealed"` takes a field whose only index is equality, on the field or on the EQL type it names. The null is sealed as long as a value and stores the null term beside it, so the absent rows are one equality group, as any repeated value is, and the column holds no SQL NULL. An `eq` query whose input is a null derives that term (`query_term`, `record::query`); a null query of any other field or operator is the new `Reason::NullQuery`. - eql-bindings: a type whose only term is equality's (`DateEq`, `IntegerEq`, ...) and its query twin take an `Option` of the plaintext, as storage-only types do, and the by-name dispatch encrypts, opens and queries them through it. - Go: stashgen defaults to `null=sealed` for a fixed-width field whose only index is equality, and generates `EqNull`, the `eq` query of nil, through the new `gensupport.Field.NullTerm`. Verified in PostgreSQL: `=` with the query of a null finds exactly the rows of a DateEq column whose value was a sealed null. Refs #1298. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql-bindings-optional-storage-only.md | 2 +- languages/golang/CHANGELOG.md | 15 +- languages/golang/cmd/stashgen/README.md | 7 +- .../golang/encrypt/gensupport/declaration.go | 8 +- languages/golang/encrypt/gensupport/field.go | 12 + .../golang/encrypt/guest/tests/native_ops.rs | 7 +- .../encrypt/internal/testusers/people.go | 24 +- .../internal/testusers/person_stash.go | 136 ++++++---- languages/golang/encrypt/null_test.go | 44 +++- languages/golang/internal/record/record.go | 5 +- languages/golang/stashgen/declaration.go | 14 +- languages/golang/stashgen/emit.go | 10 + languages/golang/stashgen/read.go | 22 +- languages/golang/stashgen/refusal_test.go | 5 +- .../stubsdk/encrypt/gensupport/gensupport.go | 5 + .../eql-bindings/src/encryption/targets.rs | 18 +- .../eql/crates/eql-bindings/src/v4/bigint.rs | 8 +- .../eql/crates/eql-bindings/src/v4/boolean.rs | 2 +- .../eql/crates/eql-bindings/src/v4/date.rs | 8 +- .../eql/crates/eql-bindings/src/v4/double.rs | 8 +- .../eql/crates/eql-bindings/src/v4/integer.rs | 8 +- .../eql/crates/eql-bindings/src/v4/numeric.rs | 8 +- .../eql/crates/eql-bindings/src/v4/real.rs | 8 +- .../crates/eql-bindings/src/v4/smallint.rs | 8 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 117 +++++---- .../eql/crates/eql-bindings/src/v4/text.rs | 8 +- .../crates/eql-bindings/src/v4/timestamp.rs | 8 +- .../eql/crates/eql-codegen/src/bindings.rs | 31 ++- .../eql/crates/eql-codegen/src/targets.rs | 18 +- .../eql/tests/encryption/tests/targets.rs | 20 +- .../encryption/tests/v4_date_time_numeric.rs | 78 ++++-- packages/stack-encrypt/CHANGELOG.md | 22 +- packages/stack-encrypt/src/dynamic/mod.rs | 14 +- packages/stack-encrypt/src/dynamic/query.rs | 18 +- packages/stack-encrypt/src/dynamic/record.rs | 233 +++++++++++++++--- packages/stack-encrypt/src/dynamic/term.rs | 45 +++- packages/stack-encrypt/src/sem/mod.rs | 34 ++- 37 files changed, 769 insertions(+), 269 deletions(-) diff --git a/.changeset/eql-bindings-optional-storage-only.md b/.changeset/eql-bindings-optional-storage-only.md index 67e8fbc35..ee48d2830 100644 --- a/.changeset/eql-bindings-optional-storage-only.md +++ b/.changeset/eql-bindings-optional-storage-only.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -`eql-bindings` (the Rust crate released with this package): a storage-only EQL v4 type (`Date`, `Integer`, `Text`, and the other types with no term) now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads and opens these types through the `Option`, so a null frame encrypts and a stored null decrypts to a null; a type with a term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. +`eql-bindings` (the Rust crate released with this package): an EQL v4 type whose terms a null has, a storage-only type (`Date`, `Integer`, `Text`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, `TextEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 4c3a02a37..747ffc732 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -20,12 +20,15 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. so the column does not show which rows have no value; `null=sql` stores nothing, so the field's columns are SQL `NULL` and its member in the generated type is a pointer. `stashgen` picks `null=sealed` for a field - with no index whose type has a fixed width (a bool, a number, - `encrypt.Date`, `time.Time`, `encrypt.Decimal`, the 128-bit integers), - and asks for `null=sql` to be written on any other: a null has no index - term yet, and text shows its length. A stored null read into a field that - is not a pointer is an error from `Decrypt`. `-model` does not map a - `null=sql` field yet. + whose type has a fixed width (a bool, a number, `encrypt.Date`, + `time.Time`, `encrypt.Decimal`, the 128-bit integers) and whose only + index, if any, is equality, and asks for `null=sql` to be written on any + other: a null has an equality term and no other, and text shows its + length. Beside equality a sealed null stores the term every nil of the + field shares, and a generated `EqNull` method derives its `eq` query, + which finds the rows whose value is absent. A stored null read into a + field that is not a pointer is an error from `Decrypt`. `-model` does not + map a `null=sql` field yet. ### Breaking diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index cb4c99fe3..a405dd62b 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -117,11 +117,12 @@ A sealed field declared over a pointer to one scalar (`*encrypt.Date`, `*int32`, | `null=` | Stored for `nil` | What a reader of the column sees | |---|---|---| -| `sealed` | a sealed null, as long as a sealed value of the field's type | nothing: an absent value looks like any other | +| `sealed` | a sealed null, as long as a sealed value of the field's type, and beside an equality index the one term every null of the field shares | nothing in the ciphertext; in an equality column, that the absent rows are one group, as rows holding one repeated value are | | `sql` | nothing: the field's columns are `NULL`, and its member in the generated type is `nil` | which rows have no value | -A sealed null hides that the value is absent only where its length matches a value's and nothing beside it shows it, so it is the default for a field with no index whose type has a fixed width: a `bool`, a number, `encrypt.Date`, `time.Time`, `encrypt.Decimal`, `encrypt.Int128` and `encrypt.Uint128`, sealed with `encrypt` or into an EQL type with no index (`Date`, `Integer`, `Boolean`, ...). -A field with an index, and a field of text or bytes, has to write `null=sql`: a null has no index term yet, and text shows its length, so a sealed null would hide nothing there, and `stashgen` will not choose a visible `NULL` for you. +A sealed null hides that the value is absent only where its length matches a value's and no index beside it shows it, so it is the default for a field whose type has a fixed width (a `bool`, a number, `encrypt.Date`, `time.Time`, `encrypt.Decimal`, `encrypt.Int128` and `encrypt.Uint128`) and whose only index, if it has one, is `equality`: sealed with `encrypt`, `encrypt,index=equality`, or into an EQL type with no index or with equality alone (`Date`, `DateEq`, `Integer`, `IntegerEq`, ...). +A null has an equality term, so its rows are one equality group and a query finds them: the generated `EqNull` derives the `eq` query of `nil`, which matches every row whose value is absent. +A null has no order or match term, so a field with any other index, and a field of text or bytes, has to write `null=sql`: an order or match column would show which values are absent, and text shows its length, and `stashgen` will not choose a visible `NULL` for you. `null=sealed` on such a field stops `stashgen` with the reason. A `null=sql` field's member in the generated type is a pointer (`*eql.DateEq`, `*EncryptedUserScore`), `nil` when the value was; a `null=sealed` field's is the same type as a non-optional field's, and its column is never `NULL`. diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index fc090762e..679927df9 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -86,8 +86,9 @@ type NullMode string const ( // NullSealed seals nil like a value, as a null as long as a value of // the field's kind, so the stored record does not show which rows are - // absent. Decrypt gives nil. The field derives no index, and its kind - // has a fixed width. + // absent. Decrypt gives nil. An equality index stores the term every nil + // shares, which [Field.NullTerm] queries; the field has no other index, + // and its kind has a fixed width. NullSealed NullMode = "sealed" // NullSQL stores nothing for nil: the field's columns hold SQL NULL, // which shows that the value is absent. Decrypt gives nil. @@ -170,7 +171,8 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara // Null says how the named field, declared over a pointer, stores nil. The // engine decides whether the field can, when the declaration is first used: -// a sealed null needs a field with no index, of a kind of fixed width. +// a sealed null needs a field whose only index is equality, of a kind of +// fixed width. func (d Declaration) Null(name string, mode NullMode) Declaration { if d.err != nil { return d diff --git a/languages/golang/encrypt/gensupport/field.go b/languages/golang/encrypt/gensupport/field.go index f4e8ebb32..b527fa505 100644 --- a/languages/golang/encrypt/gensupport/field.go +++ b/languages/golang/encrypt/gensupport/field.go @@ -100,6 +100,18 @@ func (f Field[T]) Range(ctx context.Context, c *encrypt.Cipher, operator string, return terms[0], terms[1], nil } +// NullTerm derives an exact operator's term for nil: for eq on a field that +// stores nil as a sealed null, the term every such nil shares, so it finds +// the rows whose value is absent. The engine refuses it for any other field +// or operator: nothing else stores a null's term. +func (f Field[T]) NullTerm(ctx context.Context, c *encrypt.Cipher, operator string) ([]byte, error) { + terms, err := f.query(ctx, c, operator, nil, 1) + if err != nil { + return nil, err + } + return terms[0], nil +} + // query derives an exact operator's terms, and refuses an answer of another // shape than the generated method's: a probabilistic term has a query type // of its own (Matches). Each operand is converted to the field's kind as a diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 9ab2c4d3a..eb78f343e 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -352,15 +352,16 @@ fn unsupported_term_inputs_are_encoding_errors() { let ctx = encode(s("f")); // Bools have no equality domain and no order term; match is text-only; - // containers have no term semantics; kinds outside the table and empty - // contexts are rejected. + // containers have no term semantics; a null has an equality term + // (#1298) and no order term; kinds outside the table and empty contexts + // are rejected. for (value, kind) in [ (Tree::Bool(true), TERM_EQUALITY), (Tree::Bool(true), TERM_ORE), (Tree::Bool(false), TERM_OPE), (Tree::UInt32(1), TERM_MATCH), (Tree::Array(vec![]), TERM_ORE), - (Tree::Null, TERM_EQUALITY), + (Tree::Null, TERM_ORE), (Tree::UInt32(1), 99), ] { assert_eq!( diff --git a/languages/golang/encrypt/internal/testusers/people.go b/languages/golang/encrypt/internal/testusers/people.go index 6ead84862..3255c7124 100644 --- a/languages/golang/encrypt/internal/testusers/people.go +++ b/languages/golang/encrypt/internal/testusers/people.go @@ -6,18 +6,20 @@ import "github.com/cipherstash/stack/languages/golang/encrypt" //go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Birthday -name Birthday // Person has optional fields, each declared over a pointer, in both ways a -// field stores nil (#1298): DOB and Age as a sealed null, as long as a -// value, which the generator picks for a type of fixed width with no index; -// Born, Score and Nick as SQL NULL, which an indexed field and text have to -// ask for. +// field stores nil (#1298). DOB, Born and Age store it as a sealed null, as +// long as a value, which the generator picks for a type of fixed width whose +// only index is equality: Born's and Age's equality terms of nil are the one +// term every nil shares, which EqNull queries. Joined, Score and Nick store +// SQL NULL, which an order index and text have to ask for. type Person struct { - _ struct{} `stash:"context=people"` - ID int64 `stash:"id,passthrough"` - DOB *encrypt.Date `stash:"dob,encrypt_into=Date"` - Born *encrypt.Date `stash:"born,encrypt_into=DateEq,null=sql"` - Age *int32 `stash:"age,encrypt"` - Score *int64 `stash:"score,encrypt,index=ore,null=sql"` - Nick *string `stash:"nick,encrypt,null=sql"` + _ struct{} `stash:"context=people"` + ID int64 `stash:"id,passthrough"` + DOB *encrypt.Date `stash:"dob,encrypt_into=Date"` + Born *encrypt.Date `stash:"born,encrypt_into=DateEq"` + Joined *encrypt.Date `stash:"joined,encrypt_into=DateEq,null=sql"` + Age *int32 `stash:"age,encrypt,index=equality"` + Score *int64 `stash:"score,encrypt,index=ore,null=sql"` + Nick *string `stash:"nick,encrypt,null=sql"` } // Birthday reads Person's dob column into a field that is not a pointer, diff --git a/languages/golang/encrypt/internal/testusers/person_stash.go b/languages/golang/encrypt/internal/testusers/person_stash.go index c135ea508..67b0b24b3 100644 --- a/languages/golang/encrypt/internal/testusers/person_stash.go +++ b/languages/golang/encrypt/internal/testusers/person_stash.go @@ -18,16 +18,18 @@ const _ = gensupport.GeneratedVersion1 // method. Write them, or run stashgen with -redact. type EncryptedPerson struct { - ID int64 - DOB eql.Date - Born *eql.DateEq - Age EncryptedPersonAge - Score *EncryptedPersonScore - Nick *EncryptedPersonNick + ID int64 + DOB eql.Date + Born eql.DateEq + Joined *eql.DateEq + Age EncryptedPersonAge + Score *EncryptedPersonScore + Nick *EncryptedPersonNick } type EncryptedPersonAge struct { Ciphertext encrypt.Ciphertext + Equality encrypt.EqualityTerm } type EncryptedPersonScore struct { @@ -40,24 +42,25 @@ type EncryptedPersonNick struct { } func (e EncryptedPerson) String() string { - return gensupport.Redacted("EncryptedPerson", map[string]any{"ID": e.ID}, "DOB", "Born", "Age", "Score", "Nick") + return gensupport.Redacted("EncryptedPerson", map[string]any{"ID": e.ID}, "DOB", "Born", "Joined", "Age", "Score", "Nick") } func (e EncryptedPerson) LogValue() slog.Value { - return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "DOB", "Born", "Age", "Score", "Nick") + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "DOB", "Born", "Joined", "Age", "Score", "Nick") } // Stops compiling when Person gains, loses, reorders or retypes a field. var _ = personShape(Person{}) type personShape struct { - _ struct{} - ID int64 - DOB *encrypt.Date - Born *encrypt.Date - Age *int32 - Score *int64 - Nick *string + _ struct{} + ID int64 + DOB *encrypt.Date + Born *encrypt.Date + Joined *encrypt.Date + Age *int32 + Score *int64 + Nick *string } var personDeclaration = gensupport.Declare("people"). @@ -65,8 +68,10 @@ var personDeclaration = gensupport.Declare("people"). EncryptInto("dob", gensupport.Date, "Date"). Null("dob", gensupport.NullSealed). EncryptInto("born", gensupport.Date, "DateEq"). - Null("born", gensupport.NullSQL). - Encrypt("age", gensupport.Int32). + Null("born", gensupport.NullSealed). + EncryptInto("joined", gensupport.Date, "DateEq"). + Null("joined", gensupport.NullSQL). + EncryptIndex("age", gensupport.Int32, encrypt.Equality). Null("age", gensupport.NullSealed). EncryptIndex("score", gensupport.Int64, encrypt.Ore). Null("score", gensupport.NullSQL). @@ -79,12 +84,13 @@ var personCodec = gensupport.New(gensupport.Generated[Person, EncryptedPerson]{ PrintsPlaintext: true, Source: func(v Person) gensupport.Values { return gensupport.Values{ - "id": v.ID, - "dob": gensupport.Deref(v.DOB), - "born": gensupport.Deref(v.Born), - "age": gensupport.Deref(v.Age), - "score": gensupport.Deref(v.Score), - "nick": gensupport.Deref(v.Nick), + "id": v.ID, + "dob": gensupport.Deref(v.DOB), + "born": gensupport.Deref(v.Born), + "joined": gensupport.Deref(v.Joined), + "age": gensupport.Deref(v.Age), + "score": gensupport.Deref(v.Score), + "nick": gensupport.Deref(v.Nick), } }, Seal: func(rec gensupport.Record) (EncryptedPerson, error) { @@ -94,11 +100,15 @@ var personCodec = gensupport.New(gensupport.Generated[Person, EncryptedPerson]{ return EncryptedPerson{}, err } e.DOB = eql.Date(rec["dob"].EQL) - if o, ok := rec["born"]; ok { + e.Born = eql.DateEq(rec["born"].EQL) + if o, ok := rec["joined"]; ok { v := eql.DateEq(o.EQL) - e.Born = &v + e.Joined = &v + } + e.Age = EncryptedPersonAge{ + Ciphertext: rec["age"].Ciphertext, + Equality: rec["age"].Equality, } - e.Age = EncryptedPersonAge{Ciphertext: rec["age"].Ciphertext} if o, ok := rec["score"]; ok { v := EncryptedPersonScore{ Ciphertext: o.Ciphertext, @@ -114,12 +124,13 @@ var personCodec = gensupport.New(gensupport.Generated[Person, EncryptedPerson]{ }, Open: func(e EncryptedPerson) gensupport.Record { rec := gensupport.Record{ - "id": {Value: e.ID}, - "dob": {EQL: e.DOB}, - "age": {Ciphertext: e.Age.Ciphertext}, + "id": {Value: e.ID}, + "dob": {EQL: e.DOB}, + "born": {EQL: e.Born}, + "age": {Ciphertext: e.Age.Ciphertext, Equality: e.Age.Equality}, } - if e.Born != nil { - rec["born"] = gensupport.Output{EQL: *e.Born} + if e.Joined != nil { + rec["joined"] = gensupport.Output{EQL: *e.Joined} } if e.Score != nil { rec["score"] = gensupport.Output{Ciphertext: e.Score.Ciphertext, Ore: e.Score.Ore} @@ -141,6 +152,9 @@ var personCodec = gensupport.New(gensupport.Generated[Person, EncryptedPerson]{ if v.Born, err = gensupport.GetPtr[encrypt.Date](vals, "born"); err != nil { return Person{}, err } + if v.Joined, err = gensupport.GetPtr[encrypt.Date](vals, "joined"); err != nil { + return Person{}, err + } if v.Age, err = gensupport.GetPtr[int32](vals, "age"); err != nil { return Person{}, err } @@ -168,17 +182,19 @@ func DecryptPerson(ctx context.Context, d encrypt.Decrypter, encrypted []Encrypt } var PersonFields = struct { - DOB PersonDOBField - Born PersonBornField - Age PersonAgeField - Score PersonScoreField - Nick PersonNickField + DOB PersonDOBField + Born PersonBornField + Joined PersonJoinedField + Age PersonAgeField + Score PersonScoreField + Nick PersonNickField }{ - DOB: PersonDOBField{gensupport.NewField[encrypt.Date](personDeclaration, "dob")}, - Born: PersonBornField{gensupport.NewField[encrypt.Date](personDeclaration, "born")}, - Age: PersonAgeField{gensupport.NewField[int32](personDeclaration, "age")}, - Score: PersonScoreField{gensupport.NewField[int64](personDeclaration, "score")}, - Nick: PersonNickField{gensupport.NewField[string](personDeclaration, "nick")}, + DOB: PersonDOBField{gensupport.NewField[encrypt.Date](personDeclaration, "dob")}, + Born: PersonBornField{gensupport.NewField[encrypt.Date](personDeclaration, "born")}, + Joined: PersonJoinedField{gensupport.NewField[encrypt.Date](personDeclaration, "joined")}, + Age: PersonAgeField{gensupport.NewField[int32](personDeclaration, "age")}, + Score: PersonScoreField{gensupport.NewField[int64](personDeclaration, "score")}, + Nick: PersonNickField{gensupport.NewField[string](personDeclaration, "nick")}, } type PersonDOBField struct { @@ -204,13 +220,49 @@ func (f PersonBornField) Eq(ctx context.Context, c *encrypt.Cipher, v encrypt.Da return eql.DateEqQuery(term), err } +// EqNull derives the eq query for nil: the term every stored nil of the field +// shares, so it finds the rows whose value is absent. +func (f PersonBornField) EqNull(ctx context.Context, c *encrypt.Cipher) (eql.DateEqQuery, error) { + term, err := f.field.NullTerm(ctx, c, "eq") + return eql.DateEqQuery(term), err +} + +type PersonJoinedField struct { + field gensupport.Field[encrypt.Date] +} + +func (f PersonJoinedField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.DateEq, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.DateEq(out.EQL), err +} + +func (f PersonJoinedField) Eq(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.DateEqQuery, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return eql.DateEqQuery(term), err +} + type PersonAgeField struct { field gensupport.Field[int32] } func (f PersonAgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) (EncryptedPersonAge, error) { out, err := f.field.Encrypt(ctx, c, v) - return EncryptedPersonAge{Ciphertext: out.Ciphertext}, err + if err != nil { + return EncryptedPersonAge{}, err + } + return EncryptedPersonAge{Ciphertext: out.Ciphertext, Equality: out.Equality}, nil +} + +func (f PersonAgeField) Eq(ctx context.Context, c *encrypt.Cipher, v int32) (encrypt.EqualityTerm, error) { + term, err := f.field.Term(ctx, c, "eq", v) + return encrypt.EqualityTerm(term), err +} + +// EqNull derives the eq query for nil: the term every stored nil of the field +// shares, so it finds the rows whose value is absent. +func (f PersonAgeField) EqNull(ctx context.Context, c *encrypt.Cipher) (encrypt.EqualityTerm, error) { + term, err := f.field.NullTerm(ctx, c, "eq") + return encrypt.EqualityTerm(term), err } type PersonScoreField struct { diff --git a/languages/golang/encrypt/null_test.go b/languages/golang/encrypt/null_test.go index 997745819..4a273dd1a 100644 --- a/languages/golang/encrypt/null_test.go +++ b/languages/golang/encrypt/null_test.go @@ -1,6 +1,7 @@ package encrypt_test import ( + "bytes" "context" "encoding/json" "testing" @@ -17,6 +18,18 @@ import ( func ptr[T any](v T) *T { return &v } +// hmOf is the equality term an EQL value or query holds. +func hmOf(t *testing.T, value []byte) string { + t.Helper() + var envelope struct { + HM string `json:"hm"` + } + if err := json.Unmarshal(value, &envelope); err != nil || envelope.HM == "" { + t.Fatalf("the EQL value has no hm: %v: %s", err, value) + } + return envelope.HM +} + // cOf is the length of the `c` an EQL value stores. func cOf(t *testing.T, value []byte) int { t.Helper() @@ -38,8 +51,9 @@ func TestOptionalFieldsStoreNilAsTheirTagsSay(t *testing.T) { cipher := c.DefaultKeyset() dob, born := encrypt.Date{Year: 1990, Month: 5, Day: 17}, encrypt.Date{Year: 1991, Month: 6, Day: 1} people := []testusers.Person{ - {ID: 1, DOB: &dob, Born: &born, Age: ptr(int32(34)), Score: ptr(int64(7)), Nick: ptr("al")}, + {ID: 1, DOB: &dob, Born: &born, Joined: &born, Age: ptr(int32(34)), Score: ptr(int64(7)), Nick: ptr("al")}, {ID: 2}, + {ID: 3}, } sealed, err := testusers.EncryptPerson(ctx, cipher, people) if err != nil { @@ -55,11 +69,31 @@ func TestOptionalFieldsStoreNilAsTheirTagsSay(t *testing.T) { if len(absent.Age.Ciphertext) != len(present.Age.Ciphertext) { t.Fatalf("a nil Age is sealed as long as an int32: %d bytes, beside %d", len(absent.Age.Ciphertext), len(present.Age.Ciphertext)) } + if cOf(t, absent.Born) != cOf(t, present.Born) { + t.Fatalf("a nil Born is sealed as long as a date") + } + // Beside equality, a sealed null stores the term every nil shares, and + // EqNull is its query (#1298). + other := sealed[2] + if hmOf(t, absent.Born) != hmOf(t, other.Born) || hmOf(t, absent.Born) == hmOf(t, present.Born) { + t.Fatal("every nil Born shares one term, and no date has it") + } + if !bytes.Equal(absent.Age.Equality, other.Age.Equality) || bytes.Equal(absent.Age.Equality, present.Age.Equality) { + t.Fatal("every nil Age shares one term, and no int32 has it") + } + bornNull, err := testusers.PersonFields.Born.EqNull(ctx, cipher) + if err != nil || hmOf(t, bornNull) != hmOf(t, absent.Born) { + t.Fatalf("Born.EqNull is the query of the stored nils: %v", err) + } + ageNull, err := testusers.PersonFields.Age.EqNull(ctx, cipher) + if err != nil || !bytes.Equal(ageNull, absent.Age.Equality) { + t.Fatalf("Age.EqNull is the query of the stored nils: %v", err) + } // A SQL null: nothing stored, so the column holds NULL. - if absent.Born != nil || absent.Score != nil || absent.Nick != nil { + if absent.Joined != nil || absent.Score != nil || absent.Nick != nil { t.Fatalf("a nil null=sql field stores nothing: %+v", absent) } - if present.Born == nil || present.Score == nil || present.Nick == nil { + if present.Joined == nil || present.Score == nil || present.Nick == nil { t.Fatalf("a present null=sql field is stored: %+v", present) } var stored eql.Date @@ -71,11 +105,11 @@ func TestOptionalFieldsStoreNilAsTheirTagsSay(t *testing.T) { if err != nil { t.Fatal(err) } - if got := opened[1]; got.ID != 2 || got.DOB != nil || got.Born != nil || got.Age != nil || got.Score != nil || got.Nick != nil { + if got := opened[1]; got.ID != 2 || got.DOB != nil || got.Born != nil || got.Joined != nil || got.Age != nil || got.Score != nil || got.Nick != nil { t.Fatalf("every nil decrypts to nil: %+v", got) } got := opened[0] - if got.DOB == nil || *got.DOB != dob || got.Born == nil || *got.Born != born || got.Age == nil || *got.Age != 34 || + if got.DOB == nil || *got.DOB != dob || got.Born == nil || *got.Born != born || got.Joined == nil || *got.Joined != born || got.Age == nil || *got.Age != 34 || got.Score == nil || *got.Score != 7 || got.Nick == nil || *got.Nick != "al" { t.Fatalf("every value decrypts as it was: %+v", got) } diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index c2c8db24e..9832f10dc 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -121,8 +121,9 @@ const ( NotNull Null = "" // SealedNull seals a nil like a value, as a typed null as long as one of // the field's kind, so the stored record does not show which rows are - // absent; it opens as nil. The engine takes it on a field that seals and - // derives no term, of a kind of fixed width. + // absent; it opens as nil. Beside an equality index it stores the term + // every nil shares. The engine takes it on a field whose only index is + // equality, of a kind of fixed width. SealedNull Null = "sealed" // SQLNull stores nothing for a nil: the field is left out of the record, // a source may leave it out, and a stored record without it opens it as diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index 7543f6305..d19d612e7 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -60,14 +60,16 @@ const ( // NotNull is a field that is not a pointer: it is never nil. NotNull Null = "" // NullSealed seals nil like a value, as long as one, so the stored - // field does not show that the value is absent. The default for a field - // with no index whose type has a fixed width: a bool, a number, - // encrypt.Date, time.Time, encrypt.Decimal, encrypt.Int128 or - // encrypt.Uint128. + // field does not show that the value is absent. Its equality term is the + // one every nil shares, and the generated EqNull queries it. The default + // for a field whose only index, if any, is equality and whose type has a + // fixed width: a bool, a number, encrypt.Date, time.Time, + // encrypt.Decimal, encrypt.Int128 or encrypt.Uint128. NullSealed Null = "sealed" // NullSQL stores nothing for nil: the field's columns hold SQL NULL, - // which shows that the value is absent. A field with an index, or of - // text or bytes, says so with `null=sql`: its absence would show anyway. + // which shows that the value is absent. A field with an index other than + // equality, or of text or bytes, says so with `null=sql`: its absence + // would show anyway. NullSQL Null = "sql" ) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 2cb8af7ad..e6ba5015b 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -668,6 +668,16 @@ func (w *writer) fieldEntries(f *genFile) { for _, q := range g.queries { w.nl() w.query(g, q) + if g.Null == NullSealed && q.operator.Name == "eq" && !q.operator.Pair && !q.operator.Probabilistic { + // A sealed nil stores the term every nil of the field + // shares, so eq answers for nil too. + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%sNull derives the %s query for nil: the term every stored nil of the field shares, so it finds the rows whose value is absent.", q.method, q.operator.Name), 80)) + w.p("func (f %s) %sNull(ctx context.Context, c *encrypt.Cipher) (%s, error) {", g.fieldType, q.method, q.resultType) + w.p("\tterm, err := f.field.NullTerm(ctx, c, %q)", q.operator.Name) + w.p("\treturn %s(term), err", q.resultType) + w.p("}") + } } } } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index f0a0e4765..ebbfe769b 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -793,30 +793,38 @@ func (r *reader) buildFields(c *collected) error { // nullMode is how a sealed field declared over a pointer stores nil: the // tag's `null=` if it says, else NullSealed where a sealed nil hides that the // value is absent. That needs a type of fixed width, so a nil is as long as a -// value, and no index, since a nil has no term yet (#1298) and its absence -// would show in the index column; anywhere else the field says null=sql, so -// a visible SQL NULL is a choice the source shows. +// value, and no index but equality, the one index a nil has a term for +// (#1298): any other index's column would show which values are absent. +// Anywhere else the field says null=sql, so a visible SQL NULL is a choice +// the source shows. func (r *reader) nullMode(field Field, tagged Null) (Null, error) { - indexed := len(field.Indexes) > 0 + indexes := make([]IndexName, 0, len(field.Indexes)) + for _, idx := range field.Indexes { + indexes = append(indexes, idx.Name) + } if field.Verb == VerbEncryptInto { for _, eqlType := range r.eql { if eqlType.Name == field.EQLType { - indexed = len(eqlType.Indexes) > 0 + indexes = eqlType.Indexes } } } + indexed := false + for _, name := range indexes { + indexed = indexed || name != IndexEquality + } fixed := field.GoType.Kind.fixedWidth() switch { case tagged == NullSQL: return NullSQL, nil case tagged == NullSealed && indexed: - return "", fmt.Errorf("a sealed nil has no index term yet, so the index column would show which values are absent; write null=sql, which stores SQL NULL") + return "", fmt.Errorf("a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql, which stores SQL NULL") case tagged == NullSealed && !fixed: return "", fmt.Errorf("a sealed nil is as long as a value only for a type of fixed width, and a %s shows its length; write null=sql, which stores SQL NULL", field.GoType) case fixed && !indexed: return NullSealed, nil case indexed: - return "", fmt.Errorf("a field with an index stores nil as SQL NULL, which shows which values are absent: a nil has no index term yet. Write null=sql to say so") + return "", fmt.Errorf("a field with an index other than equality stores nil as SQL NULL, which shows which values are absent: a nil has an equality term and no other. Write null=sql to say so") default: return "", fmt.Errorf("a %s field stores nil as SQL NULL, which shows which values are absent: a %s shows its length, so a sealed nil would not hide it. Write null=sql to say so", "*"+field.GoType.Name, field.GoType) } diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index de22f8402..27819cedc 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -71,9 +71,10 @@ func TestRefusals(t *testing.T) { {"null= on a passthrough", user(ctx + "\tAge *int32 `stash:\"age,passthrough,null=sql\"`"), stashgen.Request{Type: "User"}, "Age", "keeps nil as nil"}, {"an unknown null mode", user(ctx + "\tAge *int32 `stash:\"age,encrypt,null=zero\"`"), stashgen.Request{Type: "User"}, "Age", "null= is sealed or sql"}, {"null= given twice", user(ctx + "\tAge *int32 `stash:\"age,encrypt,null=sql,null=sql\"`"), stashgen.Request{Type: "User"}, "Age", "null is given twice"}, - {"an indexed pointer that does not say null=sql", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore\"`"), stashgen.Request{Type: "User"}, "Age", "Write null=sql to say so"}, + {"an ordered pointer that does not say null=sql", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore\"`"), stashgen.Request{Type: "User"}, "Age", "Write null=sql to say so"}, + {"an ordered pointer with equality that does not say null=sql", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=equality;ore\"`"), stashgen.Request{Type: "User"}, "Age", "Write null=sql to say so"}, {"a pointer to text that does not say null=sql", user(ctx + "\tNick *string `stash:\"nick,encrypt\"`"), stashgen.Request{Type: "User"}, "Nick", "a string shows its length"}, - {"a sealed null on an indexed pointer", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore,null=sealed\"`"), stashgen.Request{Type: "User"}, "Age", "a sealed nil has no index term yet"}, + {"a sealed null on an ordered pointer", user(ctx + "\tAge *int32 `stash:\"age,encrypt,index=ore,null=sealed\"`"), stashgen.Request{Type: "User"}, "Age", "a nil has an equality term and no other"}, {"a sealed null on a pointer to bytes", user(ctx + "\tBlob *[]byte `stash:\"blob,encrypt,null=sealed\"`"), stashgen.Request{Type: "User"}, "Blob", "a []byte shows its length"}, {"a match query on an index-only field", user(ctx + "\tName string `stash:\"name,index=match\"`"), stashgen.Request{Type: "User"}, "Name", "an index-only field stores no ciphertext to decrypt"}, {"a tag that does not parse", user(ctx + "\tEmail string `stash:\"email,shred\"`"), stashgen.Request{Type: "User"}, "Email", `unknown part "shred"`}, diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index eee7f4bf7..e91a1e159 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -184,5 +184,10 @@ func Deref[T any](p *T) any { return *p } +// NullTerm derives an exact operator's term for nil. +func (f Field[T]) NullTerm(context.Context, *encrypt.Cipher, string) ([]byte, error) { + return nil, nil +} + // GetPtr reads one opened field declared over a pointer. func GetPtr[T any](Values, string) (*T, error) { return nil, nil } diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 6a20a7e5c..275387466 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -545,14 +545,16 @@ mod logical { ); } -/// The plaintext of a storage-only type, which may be absent: what the -/// generated dispatch reads a storage-only type's value as, and opens it -/// into. A null seals as `None`, a typed null of the plaintext's type, as -/// long as a value when the type is fixed width (vitaminc 0.8), and opens -/// as a null; a value seals and opens exactly as the plaintext does, since -/// `Some(v)` walks `v` itself. A storage-only type has no term to derive -/// from a null. Whether a field may hold a null is the plan's to decide -/// (its `"null"`), not the type's. +/// The plaintext of a type whose terms a null has (a storage-only type, or +/// one whose only term is equality's), which may be absent: what the +/// generated dispatch reads such a type's value as, opens it into and +/// queries it with. A null seals as `None`, a typed null of the plaintext's +/// type, as long as a value when the type is fixed width (vitaminc 0.8), and +/// opens as a null; a value seals and opens exactly as the plaintext does, +/// since `Some(v)` walks `v` itself. A null's equality term is the one every +/// null shares (vitaminc-normalize 0.8), which a query of a `None` derives +/// too. Whether a field may hold a null is the plan's to decide (its +/// `"null"`), not the type's. impl Plaintext for Option { const KIND: ValueKind = S::KIND; diff --git a/packages/eql/crates/eql-bindings/src/v4/bigint.rs b/packages/eql/crates/eql-bindings/src/v4/bigint.rs index 3fedc4b73..49d1d961b 100644 --- a/packages/eql/crates/eql-bindings/src/v4/bigint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/bigint.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Bigint { doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i64`](i64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintEq { @@ -265,11 +266,12 @@ impl DomainType for BigintOrdOpe { doc = " See the [EQL bigint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`i64`](i64). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`i64`](i64). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = i64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct BigintEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/boolean.rs b/packages/eql/crates/eql-bindings/src/v4/boolean.rs index 7b3dd2237..b65be375a 100644 --- a/packages/eql/crates/eql-bindings/src/v4/boolean.rs +++ b/packages/eql/crates/eql-bindings/src/v4/boolean.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL boolean reference](https://cipherstash.com/docs/reference/eql/booleans) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`bool`](bool). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`bool`](bool). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] diff --git a/packages/eql/crates/eql-bindings/src/v4/date.rs b/packages/eql/crates/eql-bindings/src/v4/date.rs index 8651b9126..e39aadacc 100644 --- a/packages/eql/crates/eql-bindings/src/v4/date.rs +++ b/packages/eql/crates/eql-bindings/src/v4/date.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL date reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Date { doc = " See the [EQL date reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Date)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DateEq { @@ -241,11 +242,12 @@ impl DomainType for DateOrdOpe { doc = " See the [EQL date reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`Date`](crate::v4::plaintext::Date). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = crate::v4::plaintext::Date)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DateEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/double.rs b/packages/eql/crates/eql-bindings/src/v4/double.rs index 4b694e237..e5f10d4a8 100644 --- a/packages/eql/crates/eql-bindings/src/v4/double.rs +++ b/packages/eql/crates/eql-bindings/src/v4/double.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Double { doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`f64`](f64). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = f64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleEq { @@ -265,11 +266,12 @@ impl DomainType for DoubleOrdOpe { doc = " See the [EQL double reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`f64`](f64). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`f64`](f64). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = f64)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct DoubleEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/integer.rs b/packages/eql/crates/eql-bindings/src/v4/integer.rs index e7f59f7b5..9a4d49486 100644 --- a/packages/eql/crates/eql-bindings/src/v4/integer.rs +++ b/packages/eql/crates/eql-bindings/src/v4/integer.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Integer { doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i32`](i32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerEq { @@ -265,11 +266,12 @@ impl DomainType for IntegerOrdOpe { doc = " See the [EQL integer reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`i32`](i32). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`i32`](i32). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = i32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct IntegerEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/numeric.rs b/packages/eql/crates/eql-bindings/src/v4/numeric.rs index 197fccbc9..e47b394ab 100644 --- a/packages/eql/crates/eql-bindings/src/v4/numeric.rs +++ b/packages/eql/crates/eql-bindings/src/v4/numeric.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL numeric reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Numeric { doc = " See the [EQL numeric reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Decimal)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct NumericEq { @@ -241,11 +242,12 @@ impl DomainType for NumericOrdOpe { doc = " See the [EQL numeric reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`Decimal`](crate::v4::plaintext::Decimal). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = crate::v4::plaintext::Decimal)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct NumericEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/real.rs b/packages/eql/crates/eql-bindings/src/v4/real.rs index 5651100ca..8f668c900 100644 --- a/packages/eql/crates/eql-bindings/src/v4/real.rs +++ b/packages/eql/crates/eql-bindings/src/v4/real.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Real { doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`f32`](f32). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = f32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealEq { @@ -265,11 +266,12 @@ impl DomainType for RealOrdOpe { doc = " See the [EQL real reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`f32`](f32). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`f32`](f32). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = f32)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RealEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/smallint.rs b/packages/eql/crates/eql-bindings/src/v4/smallint.rs index 963a8f9be..6fe3c3ef8 100644 --- a/packages/eql/crates/eql-bindings/src/v4/smallint.rs +++ b/packages/eql/crates/eql-bindings/src/v4/smallint.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Smallint { doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`i16`](i16). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = i16)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintEq { @@ -265,11 +266,12 @@ impl DomainType for SmallintOrdOpe { doc = " See the [EQL smallint reference](https://cipherstash.com/docs/reference/eql/numbers) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`i16`](i16). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`i16`](i16). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = i16)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct SmallintEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 20ed10e19..d249b38e1 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -662,9 +662,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Integer" => run_target::, K>( "Integer", keyset, column, plaintext, ), - "IntegerEq" => { - run_target::("IntegerEq", keyset, column, plaintext) - } + "IntegerEq" => run_target::, K>( + "IntegerEq", + keyset, + column, + plaintext, + ), "IntegerOrdOre" => run_target::( "IntegerOrdOre", keyset, @@ -686,7 +689,7 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Smallint" => run_target::, K>( "Smallint", keyset, column, plaintext, ), - "SmallintEq" => run_target::( + "SmallintEq" => run_target::, K>( "SmallintEq", keyset, column, @@ -713,9 +716,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Bigint" => { run_target::, K>("Bigint", keyset, column, plaintext) } - "BigintEq" => { - run_target::("BigintEq", keyset, column, plaintext) - } + "BigintEq" => run_target::, K>( + "BigintEq", keyset, column, plaintext, + ), "BigintOrdOre" => run_target::( "BigintOrdOre", keyset, @@ -734,14 +737,16 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Date" => { run_target::, K>("Date", keyset, column, plaintext) } - "DateEq" => run_target::("DateEq", keyset, column, plaintext), + "DateEq" => { + run_target::, K>("DateEq", keyset, column, plaintext) + } "Timestamp" => run_target::, K>( "Timestamp", keyset, column, plaintext, ), - "TimestampEq" => run_target::( + "TimestampEq" => run_target::, K>( "TimestampEq", keyset, column, @@ -750,7 +755,7 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Numeric" => run_target::, K>( "Numeric", keyset, column, plaintext, ), - "NumericEq" => run_target::( + "NumericEq" => run_target::, K>( "NumericEq", keyset, column, @@ -759,9 +764,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Text" => { run_target::, K>("Text", keyset, column, plaintext) } - "TextEq" => { - run_target::("TextEq", keyset, column, plaintext) - } + "TextEq" => run_target::, K>( + "TextEq", keyset, column, plaintext, + ), "TextMatch" => { run_target::("TextMatch", keyset, column, plaintext) } @@ -798,7 +803,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Real" => { run_target::, K>("Real", keyset, column, plaintext) } - "RealEq" => run_target::("RealEq", keyset, column, plaintext), + "RealEq" => { + run_target::, K>("RealEq", keyset, column, plaintext) + } "RealOrdOre" => { run_target::("RealOrdOre", keyset, column, plaintext) } @@ -811,9 +818,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "Double" => { run_target::, K>("Double", keyset, column, plaintext) } - "DoubleEq" => { - run_target::("DoubleEq", keyset, column, plaintext) - } + "DoubleEq" => run_target::, K>( + "DoubleEq", keyset, column, plaintext, + ), "DoubleOrdOre" => run_target::( "DoubleOrdOre", keyset, @@ -844,9 +851,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Integer" => open_target::, K>( "Integer", opener, column, stored, ), - "IntegerEq" => { - open_target::("IntegerEq", opener, column, stored) - } + "IntegerEq" => open_target::, K>( + "IntegerEq", + opener, + column, + stored, + ), "IntegerOrdOre" => open_target::( "IntegerOrdOre", opener, @@ -865,9 +875,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Smallint" => open_target::, K>( "Smallint", opener, column, stored, ), - "SmallintEq" => { - open_target::("SmallintEq", opener, column, stored) - } + "SmallintEq" => open_target::, K>( + "SmallintEq", + opener, + column, + stored, + ), "SmallintOrdOre" => open_target::( "SmallintOrdOre", opener, @@ -889,9 +902,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Bigint" => { open_target::, K>("Bigint", opener, column, stored) } - "BigintEq" => { - open_target::("BigintEq", opener, column, stored) - } + "BigintEq" => open_target::, K>( + "BigintEq", opener, column, stored, + ), "BigintOrdOre" => open_target::( "BigintOrdOre", opener, @@ -908,14 +921,16 @@ pub(crate) fn decrypt_named<'a, K: 'static>( stored, ), "Date" => open_target::, K>("Date", opener, column, stored), - "DateEq" => open_target::("DateEq", opener, column, stored), + "DateEq" => { + open_target::, K>("DateEq", opener, column, stored) + } "Timestamp" => open_target::, K>( "Timestamp", opener, column, stored, ), - "TimestampEq" => open_target::( + "TimestampEq" => open_target::, K>( "TimestampEq", opener, column, @@ -924,7 +939,7 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Numeric" => open_target::, K>( "Numeric", opener, column, stored, ), - "NumericEq" => open_target::( + "NumericEq" => open_target::, K>( "NumericEq", opener, column, @@ -933,7 +948,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Text" => { open_target::, K>("Text", opener, column, stored) } - "TextEq" => open_target::("TextEq", opener, column, stored), + "TextEq" => { + open_target::, K>("TextEq", opener, column, stored) + } "TextMatch" => { open_target::("TextMatch", opener, column, stored) } @@ -959,7 +976,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Boolean", opener, column, stored, ), "Real" => open_target::, K>("Real", opener, column, stored), - "RealEq" => open_target::("RealEq", opener, column, stored), + "RealEq" => { + open_target::, K>("RealEq", opener, column, stored) + } "RealOrdOre" => { open_target::("RealOrdOre", opener, column, stored) } @@ -970,9 +989,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( "Double" => { open_target::, K>("Double", opener, column, stored) } - "DoubleEq" => { - open_target::("DoubleEq", opener, column, stored) - } + "DoubleEq" => open_target::, K>( + "DoubleEq", opener, column, stored, + ), "DoubleOrdOre" => open_target::( "DoubleOrdOre", opener, @@ -1004,7 +1023,7 @@ pub(crate) fn query_named<'a, K: 'static>( plaintext: ValueFrame, ) -> Result, K>, TargetError> { match name { - "IntegerEq" => run_target::( + "IntegerEq" => run_target::, K>( "IntegerEq", keyset, column, @@ -1028,7 +1047,7 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "SmallintEq" => run_target::( + "SmallintEq" => run_target::, K>( "SmallintEq", keyset, column, @@ -1052,7 +1071,7 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "BigintEq" => run_target::( + "BigintEq" => run_target::, K>( "BigintEq", keyset, column, plaintext, ), "BigintOrdOre" => run_target::( @@ -1073,24 +1092,24 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "DateEq" => { - run_target::("DateEq", keyset, column, plaintext) - } - "TimestampEq" => run_target::( + "DateEq" => run_target::, K>( + "DateEq", keyset, column, plaintext, + ), + "TimestampEq" => run_target::, K>( "TimestampEq", keyset, column, plaintext, ), - "NumericEq" => run_target::( + "NumericEq" => run_target::, K>( "NumericEq", keyset, column, plaintext, ), - "TextEq" => { - run_target::("TextEq", keyset, column, plaintext) - } + "TextEq" => run_target::, K>( + "TextEq", keyset, column, plaintext, + ), "TextMatch" => run_target::( "TextMatch", keyset, @@ -1154,9 +1173,9 @@ pub(crate) fn query_named<'a, K: 'static>( >("TextSearch", keyset, column, plaintext), _ => Err(no_index("TextSearch", index)), }, - "RealEq" => { - run_target::("RealEq", keyset, column, plaintext) - } + "RealEq" => run_target::, K>( + "RealEq", keyset, column, plaintext, + ), "RealOrdOre" => run_target::( "RealOrdOre", keyset, @@ -1172,7 +1191,7 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "DoubleEq" => run_target::( + "DoubleEq" => run_target::, K>( "DoubleEq", keyset, column, plaintext, ), "DoubleOrdOre" => run_target::( diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 13cc191bb..a5985740b 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Text { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEq { @@ -422,11 +423,12 @@ impl DomainType for TextSearch { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEqQuery { diff --git a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs index 072a559f7..45a95e269 100644 --- a/packages/eql/crates/eql-bindings/src/v4/timestamp.rs +++ b/packages/eql/crates/eql-bindings/src/v4/timestamp.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL timestamp reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a storage-only type carries no term, so it may store a `None`, sealed as long as a value. +/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,11 +65,12 @@ impl DomainType for Timestamp { doc = " See the [EQL timestamp reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = crate::v4::plaintext::Timestamp)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TimestampEq { @@ -241,11 +242,12 @@ impl DomainType for TimestampOrdOpe { doc = " See the [EQL timestamp reference](https://cipherstash.com/docs/reference/eql/dates-and-times) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`Timestamp`](crate::v4::plaintext::Timestamp). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = crate::v4::plaintext::Timestamp)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TimestampEqQuery { diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 17bd9654c..23cfe46ef 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -726,16 +726,24 @@ fn encryption_attrs( (false, false, true) => "order", _ => "equality", }; + let null = if nullable_terms(domain) { + format!( + ", and `EncryptFrom>`, whose `None` is the query of a stored \ + null" + ) + } else { + String::new() + }; format!( " Plaintext input: {named}. Implements `EncryptFrom<{shown}>` to build {terms} \ - query terms. Query operands do not decrypt." + query terms{null}. Query operands do not decrypt." ) - } else if domain.terms.is_empty() { + } else if nullable_terms(domain) { format!( " Plaintext input and decrypted output: {named}. Implements \ `EncryptFrom<{shown}>` and `DecryptInto<{shown}>`, and both for \ - `Option<{shown}>`: a storage-only type carries no term, so it may store a \ - `None`, sealed as long as a value." + `Option<{shown}>`: a `None` is sealed as long as a value, and its terms \ + are a null's, which equality has and no other index does." ) } else { format!( @@ -745,10 +753,10 @@ fn encryption_attrs( } }; let plaintext = plaintext_doc(&format!("[`{shown}`]({rust})")); - // A storage-only type derives no term, so an absent value has nothing - // it cannot store: it may be sealed from an `Option` too (#1298). - let nullable = (!query && domain.terms.is_empty()) - .then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); + // A type whose terms a null has (none, or equality's alone) may be + // sealed, or queried, from an `Option` too (#1298). + let nullable = + nullable_terms(domain).then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); // The twin of an optional-term query is built by its projections // (`render_query_projections`), one per index, so it carries no derive. if query && v4_optional_query_terms(domain) { @@ -787,6 +795,13 @@ fn encryption_attrs( ) } +/// Whether a null has every term the domain carries, so its type may be +/// sealed or queried from an `Option` of its plaintext (#1298): a null has +/// an equality term (`hm`, vitaminc-normalize 0.8) and no other. +pub(crate) fn nullable_terms(domain: &Domain) -> bool { + domain.terms.iter().all(|term| *term == Term::Hm) +} + /// A v4 struct's `DomainType::schema_id` overrides: the trait's defaults name /// the v3 schema base. Both forms, since the inventory's `PhantomData` handles /// reach the type-level one. diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 0b3e9935e..3d3caa1ef 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -425,11 +425,12 @@ fn render_targets_from(rows: &[Row]) -> String { let source = format_ident!("{}", rust.rsplit("::").next().unwrap_or(rust)); (name.clone(), quote!(super::#module::#ty), quote!(#source)) }; - // A storage-only type (no query twin, so no term) seals and opens an - // `Option` of its plaintext: a value exactly as the plaintext does, and - // a null as a null (#1298). Whether a field may hold one is its plan's. + // A type whose terms a null has (none, or equality's alone) seals, + // opens and queries an `Option` of its plaintext: a value exactly as the + // plaintext does, and a null as a null with a null's terms (#1298). + // Whether a field may hold one is its plan's. let stored_source = |r: &Row, source: TokenStream| { - if r.query.is_none() { + if r.indexes.iter().all(|index| *index == "eq") { quote!(Option<#source>) } else { source @@ -463,6 +464,7 @@ fn render_targets_from(rows: &[Row]) -> String { .filter_map(|r| { let (query, _) = r.query.as_ref()?; let (name, ty, source) = arm(r, query); + let source = stored_source(r, source); if r.projections.is_empty() { return Some( quote! { #name => run_target::<#ty, #source, K>(#name, keyset, column, plaintext), }, @@ -836,21 +838,19 @@ mod tests { let module = family.name; let (_, plaintext) = plaintext_of(family.name).expect("a derived family's plaintext"); let plaintext = plaintext.rsplit("::").next().unwrap(); - // A storage-only type stores and opens an `Option` of its - // plaintext; its query arm, which it has none of, would not. - let stored = if domain.terms.is_empty() { + // A type whose terms a null has (none, or equality's alone) + // stores, opens and queries an `Option` of its plaintext. + let source = if crate::bindings::nullable_terms(domain) { format!("Option<{plaintext}>") } else { plaintext.to_owned() }; let arm = |helper: &str, ty: &str| { - let source = if ty == name { &stored } else { plaintext }; format!( "\"{name}\" => {{ {helper}::(\"{name}\", " ) }; let flat_arm = |helper: &str, ty: &str| { - let source = if ty == name { &stored } else { plaintext }; format!("\"{name}\" => {helper}::(\"{name}\", ") }; // rustfmt breaks long arms at different points, so compare with diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 523ea850e..8f299bfcc 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -264,12 +264,28 @@ mod given_text_eq { &common::index_of("TextEq"), &keyset, &email(), - frame(None::), + frame(vec![34u64]), )); assert!( - matches!(error, TargetError::Plaintext { found: None, .. }), + matches!( + error, + TargetError::Plaintext { + found: Some(ValueKind::Array), + .. + } + ), "{error}" ); + // A null is a query of its own: the stored nulls' equality term + // (#1298). + assert!(targets::query( + "TextEq", + &common::index_of("TextEq"), + &keyset, + &email(), + frame(None::), + ) + .is_ok()); assert!( calls.lock().unwrap().generate.is_empty(), "nothing was minted" diff --git a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs index 55d1f741f..ccc370391 100644 --- a/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs +++ b/packages/eql/tests/encryption/tests/v4_date_time_numeric.rs @@ -973,9 +973,34 @@ async fn a_storage_only_type_stores_a_none_as_long_as_a_value() { .unwrap(); assert!(same(&opened, &null), "opens as a null by name"); - // A type with a term has no null to store: its term would show it. + // A type whose term is equality's stores a null with a null's term, + // the one a query of a null derives (#1298); a type with an order term + // has no null to store, since its term column would show it. + let eq: date::DateEq = by_name(&keyset, "DateEq", null.clone()).await; + let present_eq: date::DateEq = by_name(&keyset, "DateEq", frame(day("2024-02-29"))).await; + assert_eq!( + leaf_len(&eq.c), + leaf_len(&present_eq.c), + "as long as a value" + ); + assert_ne!(eq.hm, present_eq.hm, "a null's term is no value's"); + assert_eq!( + hm(&keyset, "DateEq", null.clone()).await, + eq.hm.0, + "a query of a null finds it" + ); + let opened = targets::decrypt( + "DateEq", + &cipher, + &label(), + &serde_json::to_vec(&eq).unwrap(), + ) + .unwrap() + .await + .unwrap(); + assert!(same(&opened, &null), "opens as a null"); assert!(matches!( - refused_null(&keyset, "DateEq"), + refused_null(&keyset, "IntegerOrd"), TargetError::Plaintext { .. } )); } @@ -1002,22 +1027,30 @@ fn postgres_stores_a_sealed_null_in_a_storage_only_column() { let keyset = cipher.default_keyset(); let mut db = database(); db.batch_execute( - "CREATE TEMP TABLE v4_optional (id int PRIMARY KEY, entry public.eql_v4_date);", + "CREATE TEMP TABLE v4_optional ( + id int PRIMARY KEY, + entry public.eql_v4_date, + due public.eql_v4_date_eq + );", ) .unwrap(); - let (absent, present): (date::Date, date::Date) = runtime.block_on(async { - ( - keyset.encrypt_as(&None::, column()).await.unwrap(), - keyset - .encrypt_as(&Date::from(day("2024-02-29")), column()) - .await - .unwrap(), - ) - }); - for (id, value) in [(1, &absent), (2, &present)] { + let rows: Vec<(i32, Option)> = vec![ + (1, None), + (2, Some(Date::from(day("2024-02-29")))), + (3, None), + ]; + for (id, value) in &rows { + let (entry, due): (date::Date, date::DateEq) = runtime.block_on(async { + ( + keyset.encrypt_as(value, column()).await.unwrap(), + keyset.encrypt_as(value, column()).await.unwrap(), + ) + }); db.execute( - "INSERT INTO v4_optional (id, entry) VALUES ($1, $2::text::jsonb::public.eql_v4_date)", - &[&id, &json(value)], + "INSERT INTO v4_optional VALUES ($1, + $2::text::jsonb::public.eql_v4_date, + $3::text::jsonb::public.eql_v4_date_eq)", + &[id, &json(&entry), &json(&due)], ) .unwrap(); } @@ -1033,4 +1066,19 @@ fn postgres_stores_a_sealed_null_in_a_storage_only_column() { .block_on(async { cipher.decrypt_as(stored, column().into()).await }) .unwrap(); assert!(opened.is_none(), "the stored null opens as None"); + + // `=` with the query of a null finds the absent rows, and only them. + let null_query: date::DateEqQuery = runtime + .block_on(async { keyset.encrypt_as(&None::, column()).await }) + .unwrap(); + let found: Vec = db + .query( + "SELECT id FROM v4_optional WHERE due = $1::text::jsonb::eql_v4.query_date_eq ORDER BY id", + &[&json(&null_query)], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(found, [1, 3]); } diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 498bafbea..a88af19ff 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -23,10 +23,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`dynamic::NullMode`, `FieldPlan::with_null`). `"sealed"` seals a null like a value, under the field's label, as vitaminc 0.8's typed null of the field's type, as long as a present value, so the stored record does - not show which rows are absent; it opens as a null. It needs a declared - type of fixed width (a bool's plain null is already as long as its value) - and no term, on the field or on the EQL type it names: refused otherwise - with `Reason::SealedNullType` or `Reason::SealedNullIndex`. `"sql"` + not show which rows are absent; it opens as a null. Beside an equality + index it stores a null's equality term, the one every null of the field + shares, so the absent rows are one equality group, as any repeated value + is, and an `eq` query whose input is a null finds them (`query_term`, + `record::query`). It needs a declared type of fixed width (a bool's plain + null is already as long as its value) and no index but equality, on the + field or on the EQL type it names, since a null has no other term: + refused otherwise with `Reason::SealedNullType` or + `Reason::SealedNullIndex`. A null query of any other field or operator is + `Reason::NullQuery`. `"sql"` stores nothing for a null, or for a row with no entry for the field: the stored record has no entry, so a host writes SQL `NULL`, and a stored record with no entry for the field opens it as a null. A batch's rows are @@ -265,10 +271,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 readers before writers; every null written before still opens. The `dynamic` feature's sinks carry a typed null through the value model, and the frame a binding receives holds the plain null. - - **A null still has no equality term.** vitaminc-normalize 0.8 derives - one; this crate drops it, so an absent value keeps a SQL `NULL` in its - equality column, as before, until the index terms of a null are decided - (#1298). + - **A null has an equality term** (vitaminc-normalize 0.8, #1298), the + one term every null of a context shares, on the typed path + (`EqualityTerm` of a `None`) and the dynamic one (`IndexSpec::Equality` + now defines a null). No other index has a term for a null. - **Block ORE's text ceiling is unchanged, but counted the new way.** `vitaminc-ore` puts the marker byte in front of text itself now, and ore-rs's 1024-byte ceiling counts it, so the fold is still at most 1023 diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index a7a72ac2c..cf6c4c1da 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -607,14 +607,20 @@ reasons! { /// A passthrough field says how to store a null: it carries one as /// it is. NullPassthrough => "null_passthrough", r#"a passthrough carries a null as it is, and takes no "null""#; - /// A field with `"null": "sealed"` derives a term, or names an EQL - /// type that carries one: a null has no term yet (#1298), so its - /// absence would show in the term column. - SealedNullIndex => "sealed_null_index", r#"a sealed null has no term yet, so a field with one takes "null": "sql""#; + /// A field with `"null": "sealed"` derives a term other than + /// equality's, or names an EQL type that carries one: a null has an + /// equality term and no other (#1298), so its absence would show in + /// that term's column. + SealedNullIndex => "sealed_null_index", r#"a null has an equality term and no other, so a field with another index takes "null": "sql""#; /// A field with `"null": "sealed"` declares no type, or one of no /// fixed width (text, bytes, a container), whose absence the length /// of its ciphertext would show. SealedNullType => "sealed_null_type", "a sealed null needs a declared type of fixed width: a bool, a number, a date, a timestamp or a decimal"; + /// A query's input is a null, and the field does not seal its nulls + /// beside an equality term: only `eq` on a `"null": "sealed"` field + /// finds a null by its term. A `"null": "sql"` field's nulls are SQL + /// `NULL`, which `IS NULL` finds. + NullQuery => "null_query", r#"a null is found by its term only with eq on a field with "null": "sealed"; a "null": "sql" field's nulls are SQL NULL"#; } } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 22ee6f526..0b1f45032 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -569,7 +569,23 @@ fn has_context(plan: &Plan, field: &str) -> Result<(), Error> { fn read_input(answer: &Answer<'_>, operator: OpSpec, input: impl Encrypt) -> Result { let name = answer.field.name(); if !operator.takes_pair() { - return read(answer.kind, input) + let input = ValueFrame::of(input).map_err(|_| Error::Source { + field: Some(name.to_owned()), + reason: Reason::FieldType, + })?; + // A null is found by its equality term, which a field that seals + // its nulls stores beside each one (#1298); nothing else has one. + if input.is_null() { + let sealed = answer.field.null() == Some(super::NullMode::Sealed); + return match (sealed, &answer.index) { + (true, IndexSpec::Equality) => Ok(Input::One(input)), + _ => Err(Error::Source { + field: Some(name.to_owned()), + reason: Reason::NullQuery, + }), + }; + } + return read(answer.kind, &input) .map(Input::One) .map_err(|error| error.in_field(name)); } diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 5021ee115..ffd6212c8 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -227,10 +227,12 @@ pub enum NullMode { /// `"sealed"`: a null is sealed like a value, under the field's label, /// as a typed null as long as a value of the field's type (vitaminc /// 0.8), so the stored record does not say which rows are absent. It - /// opens as a null. Only for a field that seals and derives no term (a - /// null has none yet), of a declared type of fixed width: a boolean, a - /// number, a date, a timestamp or a decimal. A target field takes it - /// when its EQL type carries no term. + /// opens as a null. An equality index stores a null's term beside it, + /// the one term every null of the field shares, so an `eq` query of a + /// null finds the absent rows; no other index has a term for a null. + /// Only for a field of a declared type of fixed width (a boolean, a + /// number, a date, a timestamp or a decimal) whose indexes are equality + /// alone; a target field takes it when its EQL type's are. Sealed, /// `"sql"`: a null, or a row with no entry for the field, produces /// nothing: the stored record has no entry for the field, so the host @@ -525,7 +527,11 @@ impl FieldPlan { if null == NullMode::Sql { return None; } - if !matches!(self.verb(), Verb::Encrypt | Verb::Target) { + if self + .indexes() + .iter() + .any(|index| !matches!(index, IndexSpec::Equality)) + { return Some(Reason::SealedNullIndex); } match self.field_type.and_then(ValueKind::sealed_null) { @@ -739,8 +745,14 @@ impl Plan { (None, Some(kind)) => field.field_type = Some(kind), _ => {} } - // A sealed null has no term, so its type must carry none. - if field.null == Some(NullMode::Sealed) && !descriptor.indexes.is_empty() { + // A null has an equality term and no other, so a sealed null + // needs a type whose terms are equality's alone. + if field.null == Some(NullMode::Sealed) + && descriptor + .indexes + .iter() + .any(|index| index != IndexSpec::Equality.key()) + { return Err(Error::Plan { field: Some(field.name.clone()), reason: Reason::SealedNullIndex, @@ -1085,9 +1097,11 @@ pub fn plan(frame: &[u8]) -> Result { /// value that is not there ([`NullMode`], #1298). Without it a row must /// have the field, and a typed field refuses a null. `"sealed"` seals a /// null as a typed null as long as a value, so the stored record does not -/// show it; the field must declare a type of fixed width (a bool, a number, -/// a date, a timestamp or a decimal), have no term output, and, as a target -/// field, name an EQL type with no term. `"sql"` stores nothing for a null +/// show it, and beside an equality index stores a null's equality term, the +/// one every null of the field shares, which an `eq` query of a null finds; +/// the field must declare a type of fixed width (a bool, a number, a date, a +/// timestamp or a decimal) and have no index but equality, on the field or, +/// for a target field, on its EQL type. `"sql"` stores nothing for a null /// or a missing field, so the host's columns hold SQL `NULL`; any field but /// a passthrough takes it. /// @@ -1693,7 +1707,18 @@ pub fn query<'a, K: 'static>( field: Some(field.name.clone()), reason: Reason::FieldType, })?; - check_field(&value, field)?; + // A null is queried by the term a field that seals its nulls stores + // beside each (#1298), through the type's equality index alone. + if value.is_null() { + if field.null != Some(NullMode::Sealed) || *index != IndexSpec::Equality { + return Err(Error::Source { + field: Some(field.name.clone()), + reason: Reason::NullQuery, + }); + } + } else { + check_field(&value, field)?; + } resolver .query(name, index, cipher, field.label(), value) .map_err(|error| Error::Target(name_target(error, &field.name))) @@ -2217,7 +2242,8 @@ fn source_row(mut row: Vec<(String, ValueFrame)>, plan: &Plan) -> Result { match field.field_type.and_then(ValueKind::sealed_null) { Some(Some(ty)) => ValueFrame::typed_null(ty), @@ -5286,12 +5312,17 @@ mod tests { Reason::SealedNullType, ), ( - "a sealed null beside a term", - nullable(label("f"), &["c", "eq"], Some("int64"), "sealed"), + "a sealed null beside an order term", + nullable(label("f"), &["c", "ore"], Some("int64"), "sealed"), Reason::SealedNullIndex, ), ( - "a sealed null of a term alone", + "a sealed null beside equality and an order term", + nullable(label("f"), &["c", "eq", "ore"], Some("int64"), "sealed"), + Reason::SealedNullIndex, + ), + ( + "a sealed null of an order term alone", nullable(label("f"), &["ore"], Some("int64"), "sealed"), Reason::SealedNullIndex, ), @@ -5356,6 +5387,110 @@ mod tests { assert_eq!(open(&cipher, with, &plan).await, present); } + /// The term under a stored field's output key, as its bytes. + fn stored_term(record: &StackCipherText, field: &str, key: &str) -> Vec { + let CipherText::Map(fields) = record else { + panic!("a record is a map"); + }; + let Some((_, CipherText::Map(outputs))) = fields.iter().find(|(name, _)| name == field) + else { + panic!("{field} is stored"); + }; + match outputs.iter().find(|(k, _)| k == key) { + Some((_, CipherText::Passthrough(payload))) => { + passthrough_bytes(payload).expect("a term is bytes") + } + other => panic!("{field}'s {key} is a term: {other:?}"), + } + } + + /// A null sealed beside an equality index stores the term every null + /// of the field shares (#1298): the absent rows are one equality + /// group, as any repeated value is, and an `eq` query of a null finds + /// them. The ciphertext still does not show which they are. + #[tokio::test] + async fn a_sealed_null_stores_its_equality_term_and_a_query_finds_it() { + use crate::dynamic::{query_term, NoTargets, OpSpec, Operand}; + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(obj(vec![ + ( + "age", + nullable(label("age"), &["c", "eq"], Some("uint32"), "sealed"), + ), + ( + "rank", + nullable(label("rank"), &["c", "eq", "ore"], Some("uint32"), "sql"), + ), + ])) + .expect("parses"); + let row = |age: Tree| obj(vec![("age", age), ("rank", Tree::UInt32(1))]); + let sealed = seal( + &keyset, + Tree::Array(vec![ + row(Tree::UInt32(34)), + row(Tree::Null), + row(Tree::Null), + ]), + &plan, + ) + .await; + let CipherText::Sequence(rows) = &sealed else { + panic!("a batch is a sequence"); + }; + let terms: Vec> = rows + .iter() + .map(|row| stored_term(row, "age", "eq")) + .collect(); + assert_eq!(terms[1], terms[2], "every null shares one term"); + assert_ne!(terms[0], terms[1], "and no value has it"); + assert_eq!( + sealed_len(&rows[1], "age"), + sealed_len(&rows[0], "age"), + "the ciphertext is as long as a value's" + ); + + let query = |field: &str, op: OpSpec| { + query_term(&keyset, &plan, field, op, &Tree::Null, &NoTargets) + }; + let null = query("age", OpSpec::Eq) + .expect("eq takes a null") + .await + .expect("a term"); + assert_eq!(null.operand(), &Operand::One(terms[1].clone())); + let value = query_term( + &keyset, + &plan, + "age", + OpSpec::Eq, + Tree::UInt32(34), + &NoTargets, + ) + .expect("eq takes a value") + .await + .expect("a term"); + assert_eq!(value.operand(), &Operand::One(terms[0].clone())); + + // A null has no order term, and a `"sql"` field stores none for + // its nulls, which are SQL NULL. + for (field, op) in [("rank", OpSpec::Eq), ("rank", OpSpec::Lt)] { + let refused = query(field, op).err(); + assert!( + matches!(&refused, Some(Error::Source { field: Some(f), reason: Reason::NullQuery }) if f == field), + "{field} {op}: {refused:?}" + ); + } + let opened = open(&cipher, sealed, &plan).await; + assert_eq!( + opened, + Tree::Array(vec![ + row(Tree::UInt32(34)), + row(Tree::Null), + row(Tree::Null) + ]) + ); + } + /// A stored null opens only through a plan that says the field may /// hold one: read through a plan with no `"null"`, the field's kind /// check refuses it, so a nil never reads as a zero value. @@ -6075,6 +6210,30 @@ mod tests { true, None, ), + TargetDescriptor::new( + "IntegerEq", + "integer", + "Eq", + Some(ValueKind::Int32), + "public.eql_v4_integer_eq", + vec!["eq".to_string()], + Some("IntegerEqQuery".to_string()), + Some("eql_v4.query_integer_eq".to_string()), + true, + None, + ), + TargetDescriptor::new( + "IntegerOrd", + "integer", + "Ord", + Some(ValueKind::Int32), + "public.eql_v4_integer_ord", + vec!["eq".to_string(), "ope".to_string()], + Some("IntegerOrdQuery".to_string()), + Some("eql_v4.query_integer_ord".to_string()), + true, + None, + ), TargetDescriptor::new( "TextOrdOre", "text", @@ -6494,29 +6653,45 @@ mod tests { assert_eq!(generates(&cipher), 0, "nothing minted"); } - /// A target field stores a null as its plan says: under `"sql"` any - /// type, but sealed only in a type with no term, since a null has none - /// and the type's term column would show it (#1298). Refused when the - /// plan is built, naming the field, before any value crosses. + /// A target field stores a null as its plan says (#1298): under + /// `"sql"` in any type; sealed only in a type of fixed width whose + /// terms are equality's alone, since a null has an equality term and + /// no other. Refused when the plan is built, naming the field, + /// before any value crosses. #[test] - fn a_target_field_seals_a_null_only_in_a_type_with_no_term() { - let with_null = |null: &str| { - let mut spec = target_spec(label("email"), TEXT_EQ); + fn a_target_field_seals_a_null_only_in_a_type_of_equality_alone() { + let with_null = |target: &str, null: &str| { + let mut spec = target_spec(label("f"), target); if let Tree::Object(entries) = &mut spec { entries.push(("null".to_string(), s(null))); } - obj(vec![("email", spec)]) + obj(vec![("f", spec)]) }; - let refused = plan_with(with_null("sealed"), &FakeEql).err(); + let refused = |target: &str| plan_with(with_null(target, "sealed"), &FakeEql).err(); assert!( matches!( - &refused, - Some(Error::Plan { field: Some(f), reason: Reason::SealedNullIndex }) if f == "email" + &refused("IntegerOrd"), + Some(Error::Plan { field: Some(f), reason: Reason::SealedNullIndex }) if f == "f" ), - "{refused:?}" + "an order term has no null: {:?}", + refused("IntegerOrd") ); - let parsed = plan_with(with_null("sql"), &FakeEql).expect("any type stores a SQL null"); - assert_eq!(parsed.fields()[0].null(), Some(NullMode::Sql)); + assert!( + matches!( + &refused(TEXT_EQ), + Some(Error::Plan { field: Some(f), reason: Reason::SealedNullType }) if f == "f" + ), + "text has no fixed width: {:?}", + refused(TEXT_EQ) + ); + let sealed = plan_with(with_null("IntegerEq", "sealed"), &FakeEql) + .expect("an equality type seals a null"); + assert_eq!(sealed.fields()[0].null(), Some(NullMode::Sealed)); + for target in [TEXT_EQ, "IntegerOrd"] { + let parsed = plan_with(with_null(target, "sql"), &FakeEql) + .expect("any type stores a SQL null"); + assert_eq!(parsed.fields()[0].null(), Some(NullMode::Sql)); + } } /// The resolver opens a target value through the client, which opens diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index bad1e63ba..b1c822511 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -41,7 +41,7 @@ use vitaminc_aead::plaintext::{ Error as SinkError, FixedType, Float, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, }; -use vitaminc_normalize::{domain, domain_of, normalize, NormalizeError}; +use vitaminc_normalize::{domain_of, normalize, NormalizeError}; use serde::{Deserialize, Serialize, Serializer}; @@ -150,10 +150,9 @@ impl IndexSpec { /// The table [`defines`](Self::defines) reads, over one scalar. pub(super) fn takes(&self, ty: Option, value: &Physical<'_>) -> bool { match self { - // vitaminc's table: a kind with an equality domain. A null has - // one from `vitaminc-normalize` 0.8, and no term here, as before: - // see `sem::normalized`. - IndexSpec::Equality => domain_of(ty, value).is_some_and(|d| d != domain::NULL), + // vitaminc's table: a kind with an equality domain, a null's + // among them (`vitaminc-normalize` 0.8). + IndexSpec::Equality => domain_of(ty, value).is_some(), IndexSpec::Match(_) => ty.is_none() && matches!(value, Physical::Text(_)), IndexSpec::Ore | IndexSpec::Ope | IndexSpec::OreBlock => ty.is_none() && ordered(value), } @@ -1026,6 +1025,9 @@ mod tests { ); }; } + // A null: the one term every null shares (#1298), typed or not. + equal!(None::, Tree::Null, "null"); + equal!(None::, Tree::Null, "null as text"); equal!(-3i32, Tree::Int32(-3), "i32"); equal!(-4i64, Tree::Int64(-4), "i64"); equal!(34u32, Tree::UInt32(34), "u32"); @@ -1400,7 +1402,6 @@ mod tests { let keyset = cipher.default_keyset(); let ctx = ctx("users/x"); let not_scalars = [ - ("null", Tree::Null), ("undefined", Tree::Undefined), ("an array", Tree::Array(vec![Tree::UInt32(1)])), ( @@ -1430,6 +1431,38 @@ mod tests { } } + mod given_a_null { + use super::*; + + /// A null has an equality term (`vitaminc-normalize` 0.8, #1298) and + /// no other: nothing orders or tokenizes an absent value. + #[tokio::test] + async fn it_has_an_equality_term_and_no_other() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let ctx = ctx("users/x"); + assert!(IndexSpec::Equality.defines(&Tree::Null)); + assert!( + term(&keyset, &Tree::Null, &IndexSpec::Equality, ctx.clone()) + .await + .is_ok() + ); + for kind in [ + default_match(), + IndexSpec::Ore, + IndexSpec::Ope, + IndexSpec::OreBlock, + ] { + assert!(!kind.defines(&Tree::Null), "a null has no {kind} term"); + let result = term(&keyset, &Tree::Null, &kind, ctx.clone()).await; + assert!( + matches!(&result, Err(Error::Term { kind: k, .. }) if *k == kind), + "{kind}: {result:?}" + ); + } + } + } + mod given_an_index_spec { use super::*; diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 70f5e70af..b1632a9e0 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -140,7 +140,7 @@ use clww_ore::{ClwwOpe, ClwwOre}; pub use clww_ore::{OpeClww8VariableV1, OreClww8VariableV1}; use vitaminc_aead::Encrypt; use vitaminc_hmac::HmacSha256Prf; -use vitaminc_normalize::{domain, terms, NormalizeError, Normalized}; +use vitaminc_normalize::{terms, NormalizeError, Normalized}; use vitaminc_ore::{OrderEncrypt, OrderError, OrderKeyInit, OrderScheme, OrderTarget}; use vitaminc_prf::{ BlockVisitor, Context, IntoPrfContext, MapAccess, PrfError, PrfValue, PrfVisitor, @@ -506,15 +506,11 @@ pub(crate) fn normalized(value: T) -> Result match (terms.pop(), terms.is_empty()) { // One term, and at the root: the value is that scalar. A container // with one indexable scalar inside is not a scalar, and a boolean - // produces no term at all. A null has one under `vitaminc-normalize` - // 0.8, and it is dropped here: an absent value keeps a SQL `NULL` - // in its equality column until a null's index terms are decided - // (#1298). - (Some(term), true) - if term.path.steps().is_empty() && term.normalized.domain != domain::NULL => - { - Ok(term.normalized) - } + // produces no term at all. A null, typed or plain, has one + // (`vitaminc-normalize` 0.8): every null of a context is one term, + // so an absent value sealed as a null can be stored beside its term + // and found by it (#1298). + (Some(term), true) if term.path.steps().is_empty() => Ok(term.normalized), _ => Err(TermError::NoEqualityTerm), } } @@ -2003,9 +1999,25 @@ mod tests { let prf = HmacSha256Prf::new(Protected::new([7; 32])); let derive = |value| equality(&prf, value, Context::empty()); assert!(equality(&prf, 34u32, Context::empty()).is_ok()); + // A null has one term, whatever type it stands in for and whether + // it is typed (a fixed-width type's) or plain (text's), and no value + // has it (#1298). + let null = equality(&prf, None::, Context::empty()).expect("a null has a term"); + for other in [ + equality(&prf, None::, Context::empty()), + equality(&prf, None::, Context::empty()), + equality(&prf, None::, Context::empty()), + ] { + assert_eq!(other.expect("a null has a term"), null); + } + for value in [ + equality(&prf, 0u32, Context::empty()), + equality(&prf, String::new(), Context::empty()), + ] { + assert_ne!(value.expect("a value has a term"), null); + } for (what, refused) in [ ("a boolean", equality(&prf, true, Context::empty())), - ("a null", equality(&prf, None::, Context::empty())), ( "a sequence of one", equality(&prf, vec![34u32], Context::empty()), From d0ed4a1d56ca4d5fa74db8e779f031af16cc028d Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Wed, 21 Oct 2026 21:12:21 +1100 Subject: [PATCH 07/14] test(stack-encrypt): a target query of a null needs a sealed field and equality The three survivors of cargo mutants on the null term: record::query's refusal of a null was reached by no test, since query_term refuses the same input before it. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/record.rs | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index ffd6212c8..7e9e5bf9f 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -6653,6 +6653,42 @@ mod tests { assert_eq!(generates(&cipher), 0, "nothing minted"); } + /// A null is queried by its equality term, which only a field that + /// seals its nulls stores (#1298): through such a field's equality + /// index the null reaches the resolver; through any other index, or + /// a `"sql"` field, it is refused before. (The fake resolver reads + /// text, so it refuses the null it is handed with an error of its + /// own: what matters is that the lowering let it through.) + #[tokio::test] + async fn a_target_query_of_a_null_needs_a_sealed_field_and_equality() { + let cipher = cipher().await; + let keyset = cipher.default_keyset(); + let with_null = |target: &str, null: &str| { + let mut spec = target_spec(label("f"), target); + if let Tree::Object(entries) = &mut spec { + entries.push(("null".to_string(), s(null))); + } + obj(vec![("f", spec)]) + }; + let sealed = plan_with(with_null("IntegerEq", "sealed"), &FakeEql).expect("parses"); + let sql = plan_with(with_null(TEXT_EQ, "sql"), &FakeEql).expect("parses"); + let null_query = |plan: &Plan, index: IndexSpec| { + query(&keyset, plan, "f", &index, &Tree::Null, &FakeEql).err() + }; + let refused = |error: &Option| matches!(error, Some(Error::Source { field: Some(f), reason: Reason::NullQuery }) if f == "f"); + let through = null_query(&sealed, IndexSpec::Equality); + assert!( + !refused(&through), + "a sealed field's equality takes a null: {through:?}" + ); + for (what, error) in [ + ("another index", null_query(&sealed, IndexSpec::Ore)), + ("a sql field", null_query(&sql, IndexSpec::Equality)), + ] { + assert!(refused(&error), "{what}: {error:?}"); + } + } + /// A target field stores a null as its plan says (#1298): under /// `"sql"` in any type; sealed only in a type of fixed width whose /// terms are equality's alone, since a null has an equality term and From 2e23a58fb0faf33bec56f2bcc1f669f4a4c90c5e Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 12:45:16 +1100 Subject: [PATCH 08/14] test(stack-encrypt): a sealed null is as long as a value of every fixed-width kind The length test covered a date, a float64 and a bool, so half of `ValueKind::sealed_null`'s arms ran in no test and the CRAP gate failed on it (CC 18, 47.6% covered). It now seals a value and a null of every kind the function maps, each integer and float width, a decimal, a date and a timestamp, and checks each pair is the same length and opens back. A decimal given in 64 bits is included: its null is padded to 128 bits, so the value had to be too. It is, and it opens in 128 bits. Refs #1298 Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/record.rs | 77 ++++++++++++++------ 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 7e9e5bf9f..f9a52ee7e 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -5341,50 +5341,83 @@ mod tests { /// A null sealed under `"sealed"` is a leaf as long as a value of /// the field's type, opens as a null, and costs a data key as a - /// value does. A boolean's null is the plain null, already as long. + /// value does, for every kind of fixed width. A boolean's null is + /// the plain null, already as long. A decimal held in 64 bits is + /// sealed in 128, as its null is, and opens in 128. #[tokio::test] async fn a_sealed_null_is_as_long_as_a_value_and_opens_as_a_null() { + use vitaminc_aead::plaintext::TimeUnit; let cipher = cipher().await; let keyset = cipher.default_keyset(); - let plan = plan(obj(vec![ + let typed = |logical, value| Tree::Typed(logical, Box::new(value)); + let fields = [ + ("flag", "bool", Tree::Bool(true)), + ("i8", "int8", Tree::Int8(-8)), + ("i16", "int16", Tree::Int16(-16)), + ("i32", "int32", Tree::Int32(-32)), + ("i64", "int64", Tree::Int64(-64)), + ("i128", "int128", Tree::Int128(-128)), + ("u8", "uint8", Tree::UInt8(8)), + ("u16", "uint16", Tree::UInt16(16)), + ("u32", "uint32", Tree::UInt32(32)), + ("u64", "uint64", Tree::UInt64(64)), + ("u128", "uint128", Tree::UInt128(128)), + ("f32", "float32", Tree::Float32(1.5)), + ("score", "float64", Tree::Float64(1.5)), ( - "dob", - nullable(label("dob"), &["c"], Some("date"), "sealed"), + "price", + "decimal", + typed(Logical::Decimal { scale: 2 }, Tree::Int128(150)), ), ( - "score", - nullable(label("score"), &["c"], Some("float64"), "sealed"), + "price64", + "decimal", + typed(Logical::Decimal { scale: 2 }, Tree::Int64(150)), ), + ("dob", "date", date(19_000)), ( - "flag", - nullable(label("flag"), &["c"], Some("bool"), "sealed"), + "seen", + "timestamp", + typed(Logical::Timestamp(TimeUnit::Nanos), Tree::Int128(1)), ), - ])) + ]; + let plan = plan(obj(fields + .iter() + .map(|(name, kind, _)| (*name, nullable(label(name), &["c"], Some(kind), "sealed"))) + .collect())) .expect("parses"); - let present = obj(vec![ - ("dob", date(19_000)), - ("score", Tree::Float64(1.5)), - ("flag", Tree::Bool(true)), - ]); - let absent = obj(vec![ - ("dob", Tree::Null), - ("score", Tree::Null), - ("flag", Tree::Null), - ]); + let present = obj(fields + .iter() + .map(|(name, _, value)| (*name, value.clone())) + .collect()); + let absent = obj(fields + .iter() + .map(|(name, _, _)| (*name, Tree::Null)) + .collect()); check_source(absent.clone(), &plan).expect("a null fits a field that seals one"); let with = seal(&keyset, present.clone(), &plan).await; let without = seal(&keyset, absent.clone(), &plan).await; - for field in ["dob", "score", "flag"] { + for (field, kind, _) in &fields { assert_eq!( sealed_len(&without, field), sealed_len(&with, field), - "{field}: an absent value is as long as a present one" + "{kind}: an absent value is as long as a present one" ); } let again = seal(&keyset, absent.clone(), &plan).await; check_record(again, &plan, None).expect("the stored nulls fit the plan"); + let widened = obj(fields + .iter() + .map(|(name, _, value)| match *name { + "price64" => ( + *name, + typed(Logical::Decimal { scale: 2 }, Tree::Int128(150)), + ), + _ => (*name, value.clone()), + }) + .collect()); assert_eq!(open(&cipher, without, &plan).await, absent); - assert_eq!(open(&cipher, with, &plan).await, present); + assert_eq!(open(&cipher, with, &plan).await, widened); } /// The term under a stored field's output key, as its bytes. From f3ade867b7b4e9cc3cbedaf95d86f688cec57814 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 13:46:40 +1100 Subject: [PATCH 09/14] docs(stack-encrypt): point from_bytes at the plan null mode that pads an absent value Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/value_frame.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/stack-encrypt/src/dynamic/value_frame.rs b/packages/stack-encrypt/src/dynamic/value_frame.rs index a25c36f3e..11ff10a61 100644 --- a/packages/stack-encrypt/src/dynamic/value_frame.rs +++ b/packages/stack-encrypt/src/dynamic/value_frame.rs @@ -103,7 +103,10 @@ impl ValueFrame { /// /// A typed null in the bytes reads as a plain null: the value this /// returns does not remember the type, so sealing it does not pad it to - /// the type's length. + /// the type's length. To store a field's absent values padded, give the + /// field `"null": "sealed"` in its plan ([`FieldPlan::with_null`]). + /// + /// [`FieldPlan::with_null`]: crate::dynamic::FieldPlan::with_null /// /// # Errors /// From 410db32e354e7fa433e2b399fe4a139467aa80c9 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:18:36 +1100 Subject: [PATCH 10/14] docs(stack-encrypt): say a null has an equality term IndexSpec::defines, NoEqualityTerm's doc and help, and equality_term's doc still listed a null among the values with no equality term. Since vitaminc 0.8 a null has one, the term every null shares, and defines and the term derivation already take it; only the text was stale. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- packages/stack-encrypt/src/dynamic/term.rs | 4 ++-- packages/stack-encrypt/src/sem/mod.rs | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index b1c822511..2b30b3687 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -111,8 +111,8 @@ impl IndexSpec { /// Whether the scheme defines this index's term for `value`: whether it /// is one scalar the index takes. Equality takes every scalar vitaminc - /// gives an equality domain (an integer, a float, text, bytes, and a - /// date, timestamp, decimal or uuid), not a boolean or a null. Match + /// gives an equality domain (an integer, a float, text, bytes, a date, + /// timestamp, decimal or uuid, and a null), not a boolean. Match /// takes text. ORE, OPE and block ORE take an integer of any width, a /// float, text or bytes; not a boolean, and not yet a decimal, date or /// timestamp (#1236). A match index's options do not change what it diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index b1632a9e0..ac9b4c1ab 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -255,16 +255,16 @@ pub enum TermError { )] EmptyTermText, /// The value is not one scalar an equality term is defined for: a - /// null, a boolean, a container, or a value of a logical type vitaminc - /// gives no equality domain. Equality derives over one scalar, in the - /// domain `vitaminc-normalize` gives its kind (every integer width one - /// domain, both float widths one, text, bytes, and the dates, - /// timestamps, decimals and uuids that opt in); there is no term for - /// anything else. + /// boolean, a container, or a value of a logical type vitaminc gives no + /// equality domain. Equality derives over one scalar, in the domain + /// `vitaminc-normalize` gives its kind (every integer width one domain, + /// both float widths one, text, bytes, the dates, timestamps, decimals + /// and uuids that opt in, and a null); there is no term for anything + /// else. #[error("no equality term is defined for this value")] #[diagnostic( code(stack_encrypt::no_equality_term), - help("An equality term is defined for one integer, float, text, bytes, date, timestamp, decimal or uuid value, not for a boolean, a null or a container.") + help("An equality term is defined for one integer, float, text, bytes, date, timestamp, decimal, uuid or null value, not for a boolean or a container.") )] NoEqualityTerm, /// The value cannot be normalized for its term: text with a code point @@ -1839,10 +1839,10 @@ impl KeysetCipher<'_, K> { /// PRF sees it: every integer width of one number is one term /// (`34u32` and `34i64` match), both float widths of one number are one /// term, text is compared in Unicode 16 NFC, and a date, timestamp, - /// decimal or uuid derives in a domain of its own. A boolean, a null or - /// a container has no term ([`TermError::NoEqualityTerm`]), and text with - /// a code point unassigned in Unicode 16 is refused - /// ([`TermError::Normalize`]). + /// decimal or uuid derives in a domain of its own, and every null shares + /// one term. A boolean or a container has no term + /// ([`TermError::NoEqualityTerm`]), and text with a code point + /// unassigned in Unicode 16 is refused ([`TermError::Normalize`]). pub fn equality_term<'c, T>( &self, value: T, From c7b120567b033bbec37e9608ff2349735931a999 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:18:37 +1100 Subject: [PATCH 11/14] fix(eql-bindings): seal no null of text from an Option Text and TextEq took an Option because a null has every term they carry. But a sealed null is only hidden when it is as long as a value, and text's ciphertext is as long as the text, so a None stood out beside the values the Option's docs promised it matched. The engine's plans already refuse a sealed null of text for this reason; the generated types and the by-name dispatch bypassed that check. Codegen now gives an Option plaintext only to a family of fixed width, the same rule as the plans' ValueKind::sealed_null. An optional text column stores SQL NULL. The dispatch test queries a null through IntegerEq and checks that TextEq refuses one. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql-bindings-optional-storage-only.md | 2 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 22 +++++--------- .../eql/crates/eql-bindings/src/v4/text.rs | 9 ++---- .../eql/crates/eql-codegen/src/bindings.rs | 21 ++++++++----- .../eql/crates/eql-codegen/src/targets.rs | 30 +++++++++++-------- .../eql/tests/encryption/tests/targets.rs | 26 +++++++++++++--- 6 files changed, 64 insertions(+), 46 deletions(-) diff --git a/.changeset/eql-bindings-optional-storage-only.md b/.changeset/eql-bindings-optional-storage-only.md index ee48d2830..0d1fb95ab 100644 --- a/.changeset/eql-bindings-optional-storage-only.md +++ b/.changeset/eql-bindings-optional-storage-only.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -`eql-bindings` (the Rust crate released with this package): an EQL v4 type whose terms a null has, a storage-only type (`Date`, `Integer`, `Text`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, `TextEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. +`eql-bindings` (the Rust crate released with this package): an EQL v4 type of fixed width whose terms a null has, a storage-only type (`Date`, `Integer`, `Boolean`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. Text is left out: its ciphertext is as long as the text, so a sealed null would stand out beside the values, and an optional text column stores SQL `NULL` instead. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index d249b38e1..0691ca2e9 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -761,12 +761,10 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Text" => { - run_target::, K>("Text", keyset, column, plaintext) + "Text" => run_target::("Text", keyset, column, plaintext), + "TextEq" => { + run_target::("TextEq", keyset, column, plaintext) } - "TextEq" => run_target::, K>( - "TextEq", keyset, column, plaintext, - ), "TextMatch" => { run_target::("TextMatch", keyset, column, plaintext) } @@ -945,12 +943,8 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Text" => { - open_target::, K>("Text", opener, column, stored) - } - "TextEq" => { - open_target::, K>("TextEq", opener, column, stored) - } + "Text" => open_target::("Text", opener, column, stored), + "TextEq" => open_target::("TextEq", opener, column, stored), "TextMatch" => { open_target::("TextMatch", opener, column, stored) } @@ -1107,9 +1101,9 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "TextEq" => run_target::, K>( - "TextEq", keyset, column, plaintext, - ), + "TextEq" => { + run_target::("TextEq", keyset, column, plaintext) + } "TextMatch" => run_target::( "TextMatch", keyset, diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index a5985740b..af5544826 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -15,12 +15,11 @@ use ts_rs::TS; doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] -#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Text { @@ -65,12 +64,11 @@ impl DomainType for Text { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] -#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEq { @@ -423,12 +421,11 @@ impl DomainType for TextSearch { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. +/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = String)] -#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEqQuery { diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 23cfe46ef..8704e62b4 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -726,7 +726,7 @@ fn encryption_attrs( (false, false, true) => "order", _ => "equality", }; - let null = if nullable_terms(domain) { + let null = if nullable(family.name, domain) { format!( ", and `EncryptFrom>`, whose `None` is the query of a stored \ null" @@ -738,7 +738,7 @@ fn encryption_attrs( " Plaintext input: {named}. Implements `EncryptFrom<{shown}>` to build {terms} \ query terms{null}. Query operands do not decrypt." ) - } else if nullable_terms(domain) { + } else if nullable(family.name, domain) { format!( " Plaintext input and decrypted output: {named}. Implements \ `EncryptFrom<{shown}>` and `DecryptInto<{shown}>`, and both for \ @@ -756,7 +756,7 @@ fn encryption_attrs( // A type whose terms a null has (none, or equality's alone) may be // sealed, or queried, from an `Option` too (#1298). let nullable = - nullable_terms(domain).then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); + nullable(family.name, domain).then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); // The twin of an optional-term query is built by its projections // (`render_query_projections`), one per index, so it carries no derive. if query && v4_optional_query_terms(domain) { @@ -795,11 +795,16 @@ fn encryption_attrs( ) } -/// Whether a null has every term the domain carries, so its type may be -/// sealed or queried from an `Option` of its plaintext (#1298): a null has -/// an equality term (`hm`, vitaminc-normalize 0.8) and no other. -pub(crate) fn nullable_terms(domain: &Domain) -> bool { - domain.terms.iter().all(|term| *term == Term::Hm) +/// Whether the type may be sealed or queried from an `Option` of its +/// plaintext (#1298). A null must have every term the domain carries, and +/// it has an equality term (`hm`, vitaminc-normalize 0.8) and no other. The +/// plaintext must also be of fixed width, so a sealed null is as long as a +/// value: text's ciphertext is as long as the text, so a null would stand +/// out beside it, and the engine's plans refuse a sealed null of text for +/// the same reason. An optional text column stores SQL `NULL`. +pub(crate) fn nullable(family: &str, domain: &Domain) -> bool { + let fixed_width = matches!(plaintext_of(family), Some((kind, _)) if kind != "string"); + fixed_width && domain.terms.iter().all(|term| *term == Term::Hm) } /// A v4 struct's `DomainType::schema_id` overrides: the trait's defaults name diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 3d3caa1ef..b4f2a8692 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -425,12 +425,14 @@ fn render_targets_from(rows: &[Row]) -> String { let source = format_ident!("{}", rust.rsplit("::").next().unwrap_or(rust)); (name.clone(), quote!(super::#module::#ty), quote!(#source)) }; - // A type whose terms a null has (none, or equality's alone) seals, - // opens and queries an `Option` of its plaintext: a value exactly as the - // plaintext does, and a null as a null with a null's terms (#1298). - // Whether a field may hold one is its plan's. + // A type of fixed width whose terms a null has (none, or equality's + // alone) seals, opens and queries an `Option` of its plaintext: a value + // exactly as the plaintext does, and a null as a null with a null's + // terms (#1298). Text has no fixed width, so its null is SQL's. Whether + // a field may hold one is its plan's. let stored_source = |r: &Row, source: TokenStream| { - if r.indexes.iter().all(|index| *index == "eq") { + let fixed_width = matches!(plaintext_of(r.family), Some((kind, _)) if kind != "string"); + if fixed_width && r.indexes.iter().all(|index| *index == "eq") { quote!(Option<#source>) } else { source @@ -838,9 +840,10 @@ mod tests { let module = family.name; let (_, plaintext) = plaintext_of(family.name).expect("a derived family's plaintext"); let plaintext = plaintext.rsplit("::").next().unwrap(); - // A type whose terms a null has (none, or equality's alone) - // stores, opens and queries an `Option` of its plaintext. - let source = if crate::bindings::nullable_terms(domain) { + // A type of fixed width whose terms a null has (none, or + // equality's alone) stores, opens and queries an `Option` of + // its plaintext. + let source = if crate::bindings::nullable(family.name, domain) { format!("Option<{plaintext}>") } else { plaintext.to_owned() @@ -1092,9 +1095,10 @@ mod tests { } /// A producible type with no query twin — the storage-only `Text` — - /// renders encrypt and decrypt arms over an `Option` of its plaintext, - /// so it stores a null (#1298), and no query arm: the query dispatch's - /// fall-through refuses it as answering no query. + /// renders encrypt and decrypt arms over its plaintext, and no query + /// arm: the query dispatch's fall-through refuses it as answering no + /// query. The arms take a `String`, not an `Option` of one: text has no + /// fixed width, so a sealed null would stand out beside it (#1298). #[test] fn a_producible_type_without_a_query_twin_gets_no_query_arm() { let rows = rows(); @@ -1109,12 +1113,12 @@ mod tests { .join(" "); let arms_for = |helper: &str| { out.matches(&format!( - "\"Text\" => {{ {helper}::, K>(\"Text\"," + "\"Text\" => {{ {helper}::(\"Text\"," )) .count() + out .matches(&format!( - "\"Text\" => {helper}::, K>(\"Text\"," + "\"Text\" => {helper}::(\"Text\"," )) .count() }; diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 8f299bfcc..c3d2b13bc 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -276,16 +276,34 @@ mod given_text_eq { ), "{error}" ); - // A null is a query of its own: the stored nulls' equality term - // (#1298). + // A null of a type of fixed width is a query of its own: the stored + // nulls' equality term (#1298). Text has no fixed width, so it + // stores no sealed null and its query refuses one. assert!(targets::query( + "IntegerEq", + &common::index_of("IntegerEq"), + &keyset, + &email(), + frame(None::), + ) + .is_ok()); + let error = refused(targets::query( "TextEq", &common::index_of("TextEq"), &keyset, &email(), frame(None::), - ) - .is_ok()); + )); + assert!( + matches!( + error, + TargetError::Plaintext { + target: "TextEq", + .. + } + ), + "{error}" + ); assert!( calls.lock().unwrap().generate.is_empty(), "nothing was minted" From d6e2a17e5a41ec692125dc6c749dcb110c037647 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 22:18:37 +1100 Subject: [PATCH 12/14] feat(golang): generate EncryptNull for a field that seals its nil A null=sealed field stores nil as a sealed null, a real ciphertext, so an update that sets one column to nil cannot write SQL NULL. The generated Fields entry could only seal a value, and re-encrypting the whole record is unsafe when it has index-only fields, which decrypt to zero. Each null=sealed field now has EncryptNull, through gensupport.Field.EncryptNull, which refuses a field that stores nil any other way. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- languages/golang/CHANGELOG.md | 4 +- languages/golang/cmd/stashgen/README.md | 1 + languages/golang/encrypt/gensupport/field.go | 26 ++++++++-- .../gensupport/gensupport_internal_test.go | 22 ++++++++ .../internal/testusers/person_stash.go | 27 ++++++++++ languages/golang/encrypt/null_test.go | 19 +++++++ languages/golang/stashgen/emit.go | 50 ++++++++++++------- .../stubsdk/encrypt/gensupport/gensupport.go | 5 ++ 8 files changed, 132 insertions(+), 22 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 747ffc732..c6047cf6c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -26,7 +26,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. other: a null has an equality term and no other, and text shows its length. Beside equality a sealed null stores the term every nil of the field shares, and a generated `EqNull` method derives its `eq` query, - which finds the rows whose value is absent. A stored null read into a + which finds the rows whose value is absent. A generated `EncryptNull` + method seals `nil` for an update of one such column, which writing SQL + `NULL` would not store. A stored null read into a field that is not a pointer is an error from `Decrypt`. `-model` does not map a `null=sql` field yet. diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index a405dd62b..9eb3a8198 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -122,6 +122,7 @@ A sealed field declared over a pointer to one scalar (`*encrypt.Date`, `*int32`, A sealed null hides that the value is absent only where its length matches a value's and no index beside it shows it, so it is the default for a field whose type has a fixed width (a `bool`, a number, `encrypt.Date`, `time.Time`, `encrypt.Decimal`, `encrypt.Int128` and `encrypt.Uint128`) and whose only index, if it has one, is `equality`: sealed with `encrypt`, `encrypt,index=equality`, or into an EQL type with no index or with equality alone (`Date`, `DateEq`, `Integer`, `IntegerEq`, ...). A null has an equality term, so its rows are one equality group and a query finds them: the generated `EqNull` derives the `eq` query of `nil`, which matches every row whose value is absent. +A `null=sealed` field's entry in the generated `Fields` value also has `EncryptNull`, which seals `nil` for an update of that one column: a sealed null is a value, so set the column to what it returns, not to SQL `NULL`. A null has no order or match term, so a field with any other index, and a field of text or bytes, has to write `null=sql`: an order or match column would show which values are absent, and text shows its length, and `stashgen` will not choose a visible `NULL` for you. `null=sealed` on such a field stops `stashgen` with the reason. diff --git a/languages/golang/encrypt/gensupport/field.go b/languages/golang/encrypt/gensupport/field.go index b527fa505..7e6d72746 100644 --- a/languages/golang/encrypt/gensupport/field.go +++ b/languages/golang/encrypt/gensupport/field.go @@ -50,13 +50,33 @@ func (f Field[T]) Encrypt(ctx context.Context, c *encrypt.Cipher, v T) (Output, if f.err != nil { return Output{}, f.err } - if c == nil { - return Output{}, fmt.Errorf("gensupport: %s: Encrypt needs a cipher", f.name) - } value, err := asKind(f.kind(), v) if err != nil { return Output{}, fmt.Errorf("gensupport: %s: %w", f.name, err) } + return f.seal(ctx, c, "Encrypt", value) +} + +// EncryptNull seals nil for a field that stores nil as a sealed null: a +// ciphertext as long as a value's, and the term every nil of the field +// shares. It is the update that sets one such column to nil, which writing +// SQL NULL is not. A field that stores nil any other way is refused: under +// null=sql nil is SQL NULL, and a field of no null mode has no nil. +func (f Field[T]) EncryptNull(ctx context.Context, c *encrypt.Cipher) (Output, error) { + if f.err != nil { + return Output{}, f.err + } + if pf := f.plan.Field(f.name); pf == nil || pf.Null != record.SealedNull { + return Output{}, fmt.Errorf("%w: gensupport: %s: EncryptNull seals nil only for a field that stores it as a sealed null (null=sealed)", encrypt.ErrEncoding, f.name) + } + return f.seal(ctx, c, "EncryptNull", nil) +} + +// seal seals one value of the field, or nil, through the field's own plan. +func (f Field[T]) seal(ctx context.Context, c *encrypt.Cipher, method string, value any) (Output, error) { + if c == nil { + return Output{}, fmt.Errorf("gensupport: %s: %s needs a cipher", f.name, method) + } row := record.Source{f.name: value} if f.plan.ContextField != "" { // One value of one field has no record to take its context from: diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index b39b57f65..09cc9bd1d 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -1,6 +1,7 @@ package gensupport import ( + "context" "encoding/json" "errors" "fmt" @@ -460,3 +461,24 @@ func TestAsKindLeavesAValueThatWritesItself(t *testing.T) { } } } + +// EncryptNull seals nil only for a field that stores it as a sealed null. +// Under null=sql nil is SQL NULL, and a field of no null mode has no nil, +// so both are refused before any cipher is used. +func TestEncryptNullOnlyForASealedNull(t *testing.T) { + d := Declare("people"). + EncryptIndex("age", Int32, encrypt.Equality).Null("age", NullSealed). + Encrypt("nick", String).Null("nick", NullSQL). + Encrypt("score", Int64) + for _, name := range []string{"nick", "score"} { + _, err := NewField[int64](d, name).EncryptNull(context.Background(), nil) + if !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "null=sealed") { + t.Fatalf("%s: EncryptNull = %v", name, err) + } + } + // A sealed null passes the check and reaches the cipher. + _, err := NewField[int32](d, "age").EncryptNull(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "EncryptNull needs a cipher") { + t.Fatalf("age: EncryptNull = %v", err) + } +} diff --git a/languages/golang/encrypt/internal/testusers/person_stash.go b/languages/golang/encrypt/internal/testusers/person_stash.go index 67b0b24b3..840545a05 100644 --- a/languages/golang/encrypt/internal/testusers/person_stash.go +++ b/languages/golang/encrypt/internal/testusers/person_stash.go @@ -206,6 +206,14 @@ func (f PersonDOBField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encryp return eql.Date(out.EQL), err } +// EncryptNull seals nil for DOB, as long as a value, for an update that sets +// the column to nil. Write what it returns, not SQL NULL: the field stores nil +// as a sealed null. +func (f PersonDOBField) EncryptNull(ctx context.Context, c *encrypt.Cipher) (eql.Date, error) { + out, err := f.field.EncryptNull(ctx, c) + return eql.Date(out.EQL), err +} + type PersonBornField struct { field gensupport.Field[encrypt.Date] } @@ -215,6 +223,14 @@ func (f PersonBornField) Encrypt(ctx context.Context, c *encrypt.Cipher, v encry return eql.DateEq(out.EQL), err } +// EncryptNull seals nil for Born, as long as a value, for an update that sets +// the column to nil. Write what it returns, not SQL NULL: the field stores nil +// as a sealed null. +func (f PersonBornField) EncryptNull(ctx context.Context, c *encrypt.Cipher) (eql.DateEq, error) { + out, err := f.field.EncryptNull(ctx, c) + return eql.DateEq(out.EQL), err +} + func (f PersonBornField) Eq(ctx context.Context, c *encrypt.Cipher, v encrypt.Date) (eql.DateEqQuery, error) { term, err := f.field.Term(ctx, c, "eq", v) return eql.DateEqQuery(term), err @@ -253,6 +269,17 @@ func (f PersonAgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) return EncryptedPersonAge{Ciphertext: out.Ciphertext, Equality: out.Equality}, nil } +// EncryptNull seals nil for Age, as long as a value, for an update that sets +// the column to nil. Write what it returns, not SQL NULL: the field stores nil +// as a sealed null. +func (f PersonAgeField) EncryptNull(ctx context.Context, c *encrypt.Cipher) (EncryptedPersonAge, error) { + out, err := f.field.EncryptNull(ctx, c) + if err != nil { + return EncryptedPersonAge{}, err + } + return EncryptedPersonAge{Ciphertext: out.Ciphertext, Equality: out.Equality}, nil +} + func (f PersonAgeField) Eq(ctx context.Context, c *encrypt.Cipher, v int32) (encrypt.EqualityTerm, error) { term, err := f.field.Term(ctx, c, "eq", v) return encrypt.EqualityTerm(term), err diff --git a/languages/golang/encrypt/null_test.go b/languages/golang/encrypt/null_test.go index 4a273dd1a..a1af1f9bf 100644 --- a/languages/golang/encrypt/null_test.go +++ b/languages/golang/encrypt/null_test.go @@ -89,6 +89,25 @@ func TestOptionalFieldsStoreNilAsTheirTagsSay(t *testing.T) { if err != nil || !bytes.Equal(ageNull, absent.Age.Equality) { t.Fatalf("Age.EqNull is the query of the stored nils: %v", err) } + // One column set to nil: EncryptNull seals what a whole record's nil + // seals, as long as a value, with the term every nil shares. + dobNil, err := testusers.PersonFields.DOB.EncryptNull(ctx, cipher) + if err != nil || cOf(t, dobNil) != cOf(t, present.DOB) { + t.Fatalf("DOB.EncryptNull is as long as a date: %v", err) + } + bornNil, err := testusers.PersonFields.Born.EncryptNull(ctx, cipher) + if err != nil || hmOf(t, bornNil) != hmOf(t, absent.Born) { + t.Fatalf("Born.EncryptNull stores the nils' term: %v", err) + } + ageNil, err := testusers.PersonFields.Age.EncryptNull(ctx, cipher) + if err != nil || len(ageNil.Ciphertext) != len(present.Age.Ciphertext) || !bytes.Equal(ageNil.Equality, absent.Age.Equality) { + t.Fatalf("Age.EncryptNull is as long as an int32 and stores the nils' term: %v", err) + } + updated, err := testusers.DecryptPerson(ctx, cipher, []testusers.EncryptedPerson{{ID: 4, DOB: dobNil, Born: bornNil, Age: ageNil}}) + if err != nil || updated[0].DOB != nil || updated[0].Born != nil || updated[0].Age != nil { + t.Fatalf("what EncryptNull seals decrypts to nil: %v, %+v", err, updated) + } + // A SQL null: nothing stored, so the column holds NULL. if absent.Joined != nil || absent.Score != nil || absent.Nick != nil { t.Fatalf("a nil null=sql field stores nothing: %+v", absent) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index e6ba5015b..268d1327f 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -646,25 +646,17 @@ func (w *writer) fieldEntries(f *genFile) { w.p("}") w.nl() w.p("func (f %s) Encrypt(ctx context.Context, c *encrypt.Cipher, v %s) (%s, error) {", g.fieldType, g.valueExpr, g.outputType) - switch { - case g.Verb == VerbEncryptInto: - w.p("\tout, err := f.field.Encrypt(ctx, c, v)") - w.p("\treturn %s(out.EQL), err", g.outputType) - case len(g.outputs) == 1: - w.p("\tout, err := f.field.Encrypt(ctx, c, v)") - w.p("\treturn %s{%s: out.%s}, err", g.outputType, g.outputs[0].name, g.outputs[0].name) - default: - w.p("\tout, err := f.field.Encrypt(ctx, c, v)") - w.p("\tif err != nil {") - w.p("\t\treturn %s{}, err", g.outputType) - w.p("\t}") - parts := make([]string, len(g.outputs)) - for i, o := range g.outputs { - parts[i] = fmt.Sprintf("%s: out.%s", o.name, o.name) - } - w.p("\treturn %s{%s}, nil", g.outputType, strings.Join(parts, ", ")) - } + w.sealedOutput(g, "f.field.Encrypt(ctx, c, v)") w.p("}") + if g.Null == NullSealed { + // The update of one column to nil: a sealed null is a value, + // so writing SQL NULL would not store it. + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("EncryptNull seals nil for %s, as long as a value, for an update that sets the column to nil. Write what it returns, not SQL NULL: the field stores nil as a sealed null.", g.GoName), 80)) + w.p("func (f %s) EncryptNull(ctx context.Context, c *encrypt.Cipher) (%s, error) {", g.fieldType, g.outputType) + w.sealedOutput(g, "f.field.EncryptNull(ctx, c)") + w.p("}") + } for _, q := range g.queries { w.nl() w.query(g, q) @@ -682,6 +674,28 @@ func (w *writer) fieldEntries(f *genFile) { } } +// sealedOutput writes the body of a method that seals through call, a +// gensupport.Field method returning an Output, and converts the Output to +// the field's output type. +func (w *writer) sealedOutput(g genField, call string) { + w.p("\tout, err := %s", call) + switch { + case g.Verb == VerbEncryptInto: + w.p("\treturn %s(out.EQL), err", g.outputType) + case len(g.outputs) == 1: + w.p("\treturn %s{%s: out.%s}, err", g.outputType, g.outputs[0].name, g.outputs[0].name) + default: + w.p("\tif err != nil {") + w.p("\t\treturn %s{}, err", g.outputType) + w.p("\t}") + parts := make([]string, len(g.outputs)) + for i, o := range g.outputs { + parts[i] = fmt.Sprintf("%s: out.%s", o.name, o.name) + } + w.p("\treturn %s{%s}, nil", g.outputType, strings.Join(parts, ", ")) + } +} + // queryMethod is one generated method for an operator the engine answers on // a field. type queryMethod struct { diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index e91a1e159..83abf8f0f 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -184,6 +184,11 @@ func Deref[T any](p *T) any { return *p } +// EncryptNull seals nil for a field that stores it as a sealed null. +func (f Field[T]) EncryptNull(context.Context, *encrypt.Cipher) (Output, error) { + return Output{}, nil +} + // NullTerm derives an exact operator's term for nil. func (f Field[T]) NullTerm(context.Context, *encrypt.Cipher, string) ([]byte, error) { return nil, nil From e219a942f4b9661d25b83f63936fc569806515d4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 23:00:46 +1100 Subject: [PATCH 13/14] Revert "fix(eql-bindings): seal no null of text from an Option" This reverts commit c7b120567. Text and TextEq keep their Option plaintext. A null is padded to a value's length only for types of fixed width; for text, None seals as a null of its own. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .../eql-bindings-optional-storage-only.md | 2 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 22 +++++++++----- .../eql/crates/eql-bindings/src/v4/text.rs | 9 ++++-- .../eql/crates/eql-codegen/src/bindings.rs | 21 +++++-------- .../eql/crates/eql-codegen/src/targets.rs | 30 ++++++++----------- .../eql/tests/encryption/tests/targets.rs | 26 +++------------- 6 files changed, 46 insertions(+), 64 deletions(-) diff --git a/.changeset/eql-bindings-optional-storage-only.md b/.changeset/eql-bindings-optional-storage-only.md index 0d1fb95ab..ee48d2830 100644 --- a/.changeset/eql-bindings-optional-storage-only.md +++ b/.changeset/eql-bindings-optional-storage-only.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -`eql-bindings` (the Rust crate released with this package): an EQL v4 type of fixed width whose terms a null has, a storage-only type (`Date`, `Integer`, `Boolean`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. Text is left out: its ciphertext is as long as the text, so a sealed null would stand out beside the values, and an optional text column stores SQL `NULL` instead. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. +`eql-bindings` (the Rust crate released with this package): an EQL v4 type whose terms a null has, a storage-only type (`Date`, `Integer`, `Text`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, `TextEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 0691ca2e9..d249b38e1 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -761,10 +761,12 @@ pub(crate) fn encrypt_named<'a, K: 'static>( column, plaintext, ), - "Text" => run_target::("Text", keyset, column, plaintext), - "TextEq" => { - run_target::("TextEq", keyset, column, plaintext) + "Text" => { + run_target::, K>("Text", keyset, column, plaintext) } + "TextEq" => run_target::, K>( + "TextEq", keyset, column, plaintext, + ), "TextMatch" => { run_target::("TextMatch", keyset, column, plaintext) } @@ -943,8 +945,12 @@ pub(crate) fn decrypt_named<'a, K: 'static>( column, stored, ), - "Text" => open_target::("Text", opener, column, stored), - "TextEq" => open_target::("TextEq", opener, column, stored), + "Text" => { + open_target::, K>("Text", opener, column, stored) + } + "TextEq" => { + open_target::, K>("TextEq", opener, column, stored) + } "TextMatch" => { open_target::("TextMatch", opener, column, stored) } @@ -1101,9 +1107,9 @@ pub(crate) fn query_named<'a, K: 'static>( column, plaintext, ), - "TextEq" => { - run_target::("TextEq", keyset, column, plaintext) - } + "TextEq" => run_target::, K>( + "TextEq", keyset, column, plaintext, + ), "TextMatch" => run_target::( "TextMatch", keyset, diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index af5544826..a5985740b 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -15,11 +15,12 @@ use ts_rs::TS; doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Text { @@ -64,11 +65,12 @@ impl DomainType for Text { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEq { @@ -421,11 +423,12 @@ impl DomainType for TextSearch { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms. Query operands do not decrypt. +/// Plaintext input: [`String`](String). Implements `EncryptFrom` to build equality query terms, and `EncryptFrom>`, whose `None` is the query of a stored null. Query operands do not decrypt. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom)] #[stash(plaintext = String)] +#[stash(plaintext = Option)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextEqQuery { diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 8704e62b4..23cfe46ef 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -726,7 +726,7 @@ fn encryption_attrs( (false, false, true) => "order", _ => "equality", }; - let null = if nullable(family.name, domain) { + let null = if nullable_terms(domain) { format!( ", and `EncryptFrom>`, whose `None` is the query of a stored \ null" @@ -738,7 +738,7 @@ fn encryption_attrs( " Plaintext input: {named}. Implements `EncryptFrom<{shown}>` to build {terms} \ query terms{null}. Query operands do not decrypt." ) - } else if nullable(family.name, domain) { + } else if nullable_terms(domain) { format!( " Plaintext input and decrypted output: {named}. Implements \ `EncryptFrom<{shown}>` and `DecryptInto<{shown}>`, and both for \ @@ -756,7 +756,7 @@ fn encryption_attrs( // A type whose terms a null has (none, or equality's alone) may be // sealed, or queried, from an `Option` too (#1298). let nullable = - nullable(family.name, domain).then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); + nullable_terms(domain).then(|| quote!(#[stash(plaintext = Option<#plaintext_ty>)])); // The twin of an optional-term query is built by its projections // (`render_query_projections`), one per index, so it carries no derive. if query && v4_optional_query_terms(domain) { @@ -795,16 +795,11 @@ fn encryption_attrs( ) } -/// Whether the type may be sealed or queried from an `Option` of its -/// plaintext (#1298). A null must have every term the domain carries, and -/// it has an equality term (`hm`, vitaminc-normalize 0.8) and no other. The -/// plaintext must also be of fixed width, so a sealed null is as long as a -/// value: text's ciphertext is as long as the text, so a null would stand -/// out beside it, and the engine's plans refuse a sealed null of text for -/// the same reason. An optional text column stores SQL `NULL`. -pub(crate) fn nullable(family: &str, domain: &Domain) -> bool { - let fixed_width = matches!(plaintext_of(family), Some((kind, _)) if kind != "string"); - fixed_width && domain.terms.iter().all(|term| *term == Term::Hm) +/// Whether a null has every term the domain carries, so its type may be +/// sealed or queried from an `Option` of its plaintext (#1298): a null has +/// an equality term (`hm`, vitaminc-normalize 0.8) and no other. +pub(crate) fn nullable_terms(domain: &Domain) -> bool { + domain.terms.iter().all(|term| *term == Term::Hm) } /// A v4 struct's `DomainType::schema_id` overrides: the trait's defaults name diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index b4f2a8692..3d3caa1ef 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -425,14 +425,12 @@ fn render_targets_from(rows: &[Row]) -> String { let source = format_ident!("{}", rust.rsplit("::").next().unwrap_or(rust)); (name.clone(), quote!(super::#module::#ty), quote!(#source)) }; - // A type of fixed width whose terms a null has (none, or equality's - // alone) seals, opens and queries an `Option` of its plaintext: a value - // exactly as the plaintext does, and a null as a null with a null's - // terms (#1298). Text has no fixed width, so its null is SQL's. Whether - // a field may hold one is its plan's. + // A type whose terms a null has (none, or equality's alone) seals, + // opens and queries an `Option` of its plaintext: a value exactly as the + // plaintext does, and a null as a null with a null's terms (#1298). + // Whether a field may hold one is its plan's. let stored_source = |r: &Row, source: TokenStream| { - let fixed_width = matches!(plaintext_of(r.family), Some((kind, _)) if kind != "string"); - if fixed_width && r.indexes.iter().all(|index| *index == "eq") { + if r.indexes.iter().all(|index| *index == "eq") { quote!(Option<#source>) } else { source @@ -840,10 +838,9 @@ mod tests { let module = family.name; let (_, plaintext) = plaintext_of(family.name).expect("a derived family's plaintext"); let plaintext = plaintext.rsplit("::").next().unwrap(); - // A type of fixed width whose terms a null has (none, or - // equality's alone) stores, opens and queries an `Option` of - // its plaintext. - let source = if crate::bindings::nullable(family.name, domain) { + // A type whose terms a null has (none, or equality's alone) + // stores, opens and queries an `Option` of its plaintext. + let source = if crate::bindings::nullable_terms(domain) { format!("Option<{plaintext}>") } else { plaintext.to_owned() @@ -1095,10 +1092,9 @@ mod tests { } /// A producible type with no query twin — the storage-only `Text` — - /// renders encrypt and decrypt arms over its plaintext, and no query - /// arm: the query dispatch's fall-through refuses it as answering no - /// query. The arms take a `String`, not an `Option` of one: text has no - /// fixed width, so a sealed null would stand out beside it (#1298). + /// renders encrypt and decrypt arms over an `Option` of its plaintext, + /// so it stores a null (#1298), and no query arm: the query dispatch's + /// fall-through refuses it as answering no query. #[test] fn a_producible_type_without_a_query_twin_gets_no_query_arm() { let rows = rows(); @@ -1113,12 +1109,12 @@ mod tests { .join(" "); let arms_for = |helper: &str| { out.matches(&format!( - "\"Text\" => {{ {helper}::(\"Text\"," + "\"Text\" => {{ {helper}::, K>(\"Text\"," )) .count() + out .matches(&format!( - "\"Text\" => {helper}::(\"Text\"," + "\"Text\" => {helper}::, K>(\"Text\"," )) .count() }; diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index c3d2b13bc..8f299bfcc 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -276,34 +276,16 @@ mod given_text_eq { ), "{error}" ); - // A null of a type of fixed width is a query of its own: the stored - // nulls' equality term (#1298). Text has no fixed width, so it - // stores no sealed null and its query refuses one. + // A null is a query of its own: the stored nulls' equality term + // (#1298). assert!(targets::query( - "IntegerEq", - &common::index_of("IntegerEq"), - &keyset, - &email(), - frame(None::), - ) - .is_ok()); - let error = refused(targets::query( "TextEq", &common::index_of("TextEq"), &keyset, &email(), frame(None::), - )); - assert!( - matches!( - error, - TargetError::Plaintext { - target: "TextEq", - .. - } - ), - "{error}" - ); + ) + .is_ok()); assert!( calls.lock().unwrap().generate.is_empty(), "nothing was minted" From e6dc8f9a5e56027b6e066cf9aeb08feedfd99a34 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 23 Oct 2026 23:08:50 +1100 Subject: [PATCH 14/14] docs(eql-bindings): say a text null is not padded The generated docs for Text and TextEq said a None is sealed as long as a value. That holds for types of fixed width only: a text null is not padded, since text's ciphertext is as long as the text, so its length may show that the value is absent. The changeset says the same. Claude-Session: https://claude.ai/code/session_01XQN1QpaWVDGjHEg39KEJF3 --- .changeset/eql-bindings-optional-storage-only.md | 2 +- packages/eql/crates/eql-bindings/src/v4/text.rs | 4 ++-- packages/eql/crates/eql-codegen/src/bindings.rs | 12 ++++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.changeset/eql-bindings-optional-storage-only.md b/.changeset/eql-bindings-optional-storage-only.md index ee48d2830..17b2f3f17 100644 --- a/.changeset/eql-bindings-optional-storage-only.md +++ b/.changeset/eql-bindings-optional-storage-only.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -`eql-bindings` (the Rust crate released with this package): an EQL v4 type whose terms a null has, a storage-only type (`Date`, `Integer`, `Text`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, `TextEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. A `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent, and opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. +`eql-bindings` (the Rust crate released with this package): an EQL v4 type whose terms a null has, a storage-only type (`Date`, `Integer`, `Text`, ...) or one whose only term is equality's (`DateEq`, `IntegerEq`, `TextEq`, ...), now also implements `EncryptFrom` and `DecryptInto` for an `Option` of its plaintext, and its query twin `EncryptFrom` for one. For a type of fixed width a `None` seals as a typed null as long as a value (vitaminc 0.8), so an optional column does not show which rows are absent. For text it seals as a null that is not padded, since text's ciphertext is as long as the text, so its length may show that the value is absent. Either way it opens as `None`; its `hm` is the one term every null shares, and the query of a `None` finds the stored nulls with `=`. A value seals exactly as before. The by-name dispatch (`encryption::targets`) reads, opens and queries these types through the `Option`; a type with an order or match term still refuses a null. eql-bindings' own `Date`, `Timestamp` and `Decimal` plaintexts declare their fixed width to vitaminc, so their `None` is padded too. The SQL bundle and the TypeScript types do not change. diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index a5985740b..3314d75ad 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -15,7 +15,7 @@ use ts_rs::TS; doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as a null that is not padded, since text has no one length, so its length may show that the value is absent, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] @@ -65,7 +65,7 @@ impl DomainType for Text { doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] /// -/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as long as a value, and its terms are a null's, which equality has and no other index does. +/// Plaintext input and decrypted output: [`String`](String). Implements `EncryptFrom` and `DecryptInto`, and both for `Option`: a `None` is sealed as a null that is not padded, since text has no one length, so its length may show that the value is absent, and its terms are a null's, which equality has and no other index does. /// Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`. /// See the [complete encryption example](crate::encryption#example). #[derive(stack_encrypt::EncryptFrom, stack_encrypt::DecryptInto)] diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 23cfe46ef..4616a53fb 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -739,11 +739,19 @@ fn encryption_attrs( query terms{null}. Query operands do not decrypt." ) } else if nullable_terms(domain) { + // A null is padded to a value's length only where values have + // one length; text's ciphertext is as long as the text. + let length = if family.name == "text" { + "a `None` is sealed as a null that is not padded, since text has no \ + one length, so its length may show that the value is absent" + } else { + "a `None` is sealed as long as a value" + }; format!( " Plaintext input and decrypted output: {named}. Implements \ `EncryptFrom<{shown}>` and `DecryptInto<{shown}>`, and both for \ - `Option<{shown}>`: a `None` is sealed as long as a value, and its terms \ - are a null's, which equality has and no other index does." + `Option<{shown}>`: {length}, and its terms are a null's, which equality \ + has and no other index does." ) } else { format!(