Store and serve the session-level data container

README.md

@@ -146,6 +146,23 @@ instead of rejection — enum synonyms, fields withdrawn on privacy grounds,
 turn content above the declared privacy level — the server normalises and logs
 what it changed.
 
+The session-level `data` container (section 5.1.3) is stored and served back
+unchanged, `access_context` included. Only its shape is checked:
+`identifiers` must be an array of objects each carrying a string `scheme` and
+a string `value`. The scheme vocabulary is open, so a scheme outside the core
+`ror`, `saml_entity_id` and `isni` is stored like any other, as are unknown
+members alongside `access_context` and inside it.
+
+Top-level members of a session document that this server does not define are
+recorded rather than dropped. The session root is not an extension point
+(section 5.1.3), so nothing there is interpreted, but a consumer must tolerate
+unknown fields without error (section 5.7.4), which rules out refusing the
+document. Ingest logs the member names and stores them on the session, and
+materialisation returns them under `extensions.unrecognised_fields`. A field
+the specification adds that this server does not implement yet is therefore
+visible in the log and in the document, rather than accepted into a success
+response and discarded.
+
 Materialising a session document keeps that honest in the other direction.
 Events that predate v1 or belong to an extension move under the document's
 `extensions` member rather than being dropped or rewritten, so the document

crates/content-telemetry-core/migrations/0004_session_data.sql

@@ -0,0 +1,20 @@
+-- The session-level data container, and the top-level members the
+-- specification does not define.
+--
+-- session_data (specification section 5.1.3): the session document's `data`
+-- object, an extension container mirroring the event-level `data` field.
+-- Core defines one member inside it, `access_context`, which records the
+-- institution whose access rights the session used. Consumers MUST tolerate
+-- unknown fields within the container and unknown identifier schemes within
+-- access_context, so the column holds what the emitter sent, unnormalised,
+-- and materialisation serves it back unchanged.
+--
+-- unrecognised_fields (specification sections 5.1.3, 5.7.4): top-level
+-- siblings of `events` that this server does not define. The session root is
+-- not an extension point, so nothing here is interpreted, but a conforming
+-- consumer MUST tolerate unknown fields without error - so they are recorded
+-- instead of being dropped, and materialisation returns them under the
+-- document's `extensions` member.
+
+ALTER TABLE sessions ADD COLUMN IF NOT EXISTS session_data JSONB;
+ALTER TABLE sessions ADD COLUMN IF NOT EXISTS unrecognised_fields JSONB;

crates/content-telemetry-core/src/conformance.rs

@@ -84,6 +84,17 @@ pub const WITHDRAWN_EVENT_TYPE_DISPLAYED: &str = "content_displayed";
 /// v1 transition rule, not a general registry of withdrawn extension names.
 pub const WITHDRAWN_EVENT_DATA_FIELDS: &[&str] = &["ip_hash"];
 
+/// The one container core defines inside the session-level `data` object
+/// (spec 5.1.3): the context from which the session's access rights derive,
+/// an institution and never an individual. COUNTER usage reporting needs it,
+/// which is why it is in core rather than in an extension.
+pub const SESSION_ACCESS_CONTEXT: &str = "access_context";
+
+/// Identifier schemes named in core (spec 5.1.3). Informative only: the
+/// vocabulary is open, emitters MAY use others, and consumers MUST tolerate
+/// unknown ones, so nothing validates against this list.
+pub const CORE_ACCESS_CONTEXT_SCHEMES: &[&str] = &["ror", "saml_entity_id", "isni"];
+
 /// Conversation-turn privacy levels (spec 5.4). A closed enum: the schema
 /// rejects a turn whose `privacy_level` is outside this set.
 pub const PRIVACY_LEVELS: &[&str] = &["full", "summary", "intent", "minimal"];
@@ -380,6 +391,68 @@ pub fn v1_structural_violation(
     }
 }
 
+/// The shape the schema gives the session-level `data` container's one core
+/// member, `access_context` (spec 5.1.3): `identifiers` is an array, and
+/// every entry is an object carrying a `scheme` and a `value`, both strings.
+///
+/// Only the shape is checked. Scheme values are open — `ror`,
+/// `saml_entity_id` and `isni` are the core ones and a consumer MUST
+/// tolerate any other — and additional members inside `data`, inside
+/// `access_context` and inside an identifier are tolerated unchanged,
+/// because consumers MUST tolerate unknown fields within the container.
+///
+/// Returns a description of the first violation, or None.
+pub fn session_data_violation(data: Option<&Value>) -> Option<String> {
+    let data = data?;
+    if data.is_null() {
+        return None;
+    }
+    let Some(data) = data.as_object() else {
+        return Some("session data must be an object".to_string());
+    };
+
+    let access_context = data.get(SESSION_ACCESS_CONTEXT)?;
+    if access_context.is_null() {
+        return None;
+    }
+    let Some(access_context) = access_context.as_object() else {
+        return Some("data.access_context must be an object".to_string());
+    };
+
+    let identifiers = access_context.get("identifiers")?;
+    let Some(identifiers) = identifiers.as_array() else {
+        return Some(
+            "data.access_context.identifiers must be an array of {scheme, value} objects"
+                .to_string(),
+        );
+    };
+
+    for (index, identifier) in identifiers.iter().enumerate() {
+        let Some(identifier) = identifier.as_object() else {
+            return Some(format!(
+                "data.access_context.identifiers[{index}] must be an object"
+            ));
+        };
+        for member in ["scheme", "value"] {
+            match identifier.get(member) {
+                Some(v) if v.is_string() => {}
+                Some(_) => {
+                    return Some(format!(
+                        "data.access_context.identifiers[{index}].{member} must be a string"
+                    ));
+                }
+                None => {
+                    return Some(format!(
+                        "data.access_context.identifiers[{index}] requires {member}"
+                    ));
+                }
+            }
+        }
+    }
+
+    None
+}
+
 /// The v1 rule that `source_role` MUST be present on every
 /// `content_retrieved` event (spec 5.2.2, 5.7.5): without it a consumer
 /// cannot tell an agent-reported fetch from an origin- or edge-reported
@@ -1146,4 +1219,69 @@ mod tests {
         });
         assert!(strip_turn_privacy_violations(&mut turn).is_empty());
     }
+
+    #[test]
+    fn access_context_shape_is_checked_and_its_vocabulary_is_not() {
+        // The spec's own example (5.1.3), plus a scheme core does not name.
+        // Consumers MUST tolerate unknown schemes, so the unknown one is not
+        // a violation.
+        let spec_example = json!({
+            "access_context": {
+                "identifiers": [
+                    { "scheme": "ror", "value": "https://ror.org/013meh722" },
+                    { "scheme": "saml_entity_id", "value": "https://idp.example.ac.uk/shibboleth" },
+                    { "scheme": "example_local", "value": "lib-4471" }
+                ]
+            }
+        });
+        assert!(session_data_violation(Some(&spec_example)).is_none());
+
+        // Unknown fields inside the container, inside access_context and
+        // inside an identifier are all tolerated (5.1.3).
+        let extras = json!({
+            "com.example.reporting_period": "2026-08",
+            "access_context": {
+                "asserted_by": "agent",
+                "identifiers": [
+                    { "scheme": "isni", "value": "0000000121032683", "note": "consortium seat" }
+                ]
+            }
+        });
+        assert!(session_data_violation(Some(&extras)).is_none());
+
+        // A container with no access_context, and no container at all.
+        assert!(session_data_violation(Some(&json!({ "x": 1 }))).is_none());
+        assert!(session_data_violation(None).is_none());
+        assert!(session_data_violation(Some(&Value::Null)).is_none());
+    }
+
+    #[test]
+    fn malformed_access_context_is_a_violation() {
+        // Both cases are the standard's own invalid fixtures.
+        let missing_value = json!({
+            "access_context": { "identifiers": [{ "scheme": "ror" }] }
+        });
+        assert_eq!(
+            session_data_violation(Some(&missing_value)).as_deref(),
+            Some("data.access_context.identifiers[0] requires value")
+        );
+
+        let not_an_array = json!({
+            "access_context": { "identifiers": "https://ror.org/013meh722" }
+        });
+        assert!(
+            session_data_violation(Some(&not_an_array))
+                .is_some_and(|v| v.contains("must be an array"))
+        );
+
+        let non_string_scheme = json!({
+            "access_context": { "identifiers": [{ "scheme": 7, "value": "x" }] }
+        });
+        assert_eq!(
+            session_data_violation(Some(&non_string_scheme)).as_deref(),
+            Some("data.access_context.identifiers[0].scheme must be a string")
+        );
+
+        assert!(session_data_violation(Some(&json!("not an object"))).is_some());
+    }
 }

crates/content-telemetry-core/src/models/session.rs

@@ -26,6 +26,12 @@ pub struct SessionCreateRequest {
     pub conformance_level: Option<String>,
     pub agent_id: Option<String>,
     pub external_session_id: Option<String>,
+    /// Session-level extension container (spec 5.1.3), carrying
+    /// `access_context` and any namespaced members alongside it. Stored and
+    /// served verbatim: consumers MUST tolerate unknown fields within it and
+    /// unknown identifier schemes inside `access_context`, so nothing here
+    /// is normalised or dropped.
+    pub data: Option<serde_json::Value>,
     #[serde(default)]
     pub user_context: serde_json::Value,
     #[serde(default)]
@@ -38,6 +44,13 @@ pub struct SessionCreateRequest {
     /// ingest. Defaults to NOW() when absent (the /sessions/start flow).
     pub started_at: Option<DateTime<Utc>>,
     pub ended_at: Option<DateTime<Utc>>,
+    /// Top-level members this server does not define. The session root is
+    /// not an extension point (spec 5.1.3), so nothing here is interpreted,
+    /// but a conforming consumer MUST tolerate unknown fields without error
+    /// (spec 5.7.4). Capturing them is what stops a member the specification
+    /// adds later from being accepted and silently discarded.
+    #[serde(flatten)]
+    pub unrecognised_fields: serde_json::Map<String, serde_json::Value>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -90,6 +103,10 @@ pub struct BulkSessionRequest {
     pub prior_session_ids: Vec<Uuid>,
     pub started_at: Option<DateTime<Utc>>,
     pub ended_at: Option<DateTime<Utc>>,
+    /// Session-level extension container (spec 5.1.3), carrying
+    /// `access_context` and any namespaced members alongside it. Stored and
+    /// served verbatim.
+    pub data: Option<serde_json::Value>,
     #[serde(default)]
     pub user_context: serde_json::Value,
     #[serde(default)]
@@ -99,6 +116,56 @@ pub struct BulkSessionRequest {
     pub platform_id: Option<String>,
     pub client_type: Option<String>,
     pub client_info: Option<serde_json::Value>,
+    /// Top-level members this server does not define (spec 5.1.3, 5.7.4).
+    /// Recorded rather than dropped; see `SessionCreateRequest`.
+    #[serde(flatten)]
+    pub unrecognised_fields: serde_json::Map<String, serde_json::Value>,
+}
+
+impl BulkSessionRequest {
+    /// The session this document opens, as the bulk ingest path creates it.
+    ///
+    /// One mapping from the document format onto the stored session, rather
+    /// than a hand-written copy in each handler: a member the format gains
+    /// is either carried here or visibly absent here, never dropped in a
+    /// place nobody looks.
+    ///
+    /// `conformance_level` passes through unnormalised — `create_session`
+    /// normalises it (spec 5.7) — and `ended_at` is withheld when the
+    /// document carries an outcome, because ending a session only updates a
+    /// row whose `ended_at` is still NULL and stamping it here would drop
+    /// the outcome. The caller hands that timestamp to `end_session`.
+    pub fn session_create(&self) -> SessionCreateRequest {
+        SessionCreateRequest {
+            initiator_type: self.initiator_type.clone(),
+            initiator: self.initiator.clone(),
+            parent_session_id: self.parent_session_id,
+            content_scope: self.content_scope.clone(),
+            manifest_ref: self.manifest_ref.clone(),
+            conformance_level: self.conformance_level.clone(),
+            agent_id: self.agent_id.clone(),
+            // The presented session id is the emitter's, not ours: it is
+            // stored as the external id under a server-minted primary key.
+            external_session_id: Some(
+                self.external_session_id
+                    .clone()
+                    .unwrap_or_else(|| self.session_id.to_string()),
+            ),
+            data: self.data.clone(),
+            user_context: self.user_context.clone(),
+            prior_session_ids: self.prior_session_ids.iter().map(Uuid::to_string).collect(),
+            platform_id: self.platform_id.clone(),
+            client_type: self.client_type.clone(),
+            client_info: self.client_info.clone(),
+            started_at: self.started_at,
+            ended_at: if self.outcome.is_some() {
+                None
+            } else {
+                self.ended_at
+            },
+            unrecognised_fields: self.unrecognised_fields.clone(),
+        }
+    }
 }
 
 // ---------------------------------------------------------------------------
@@ -119,6 +186,12 @@ pub struct SessionRow {
     pub agent_id: Option<String>,
     pub external_session_id: Option<String>,
     pub prior_session_ids: Option<Vec<Uuid>>,
+    /// The session-level `data` container as the emitter sent it
+    /// (spec 5.1.3).
+    pub session_data: Option<serde_json::Value>,
+    /// Top-level members the specification does not define, kept so nothing
+    /// a conformant document carries is lost without trace (spec 5.7.4).
+    pub unrecognised_fields: Option<serde_json::Value>,
     pub user_context: serde_json::Value,
     pub platform_id: Option<String>,
     pub client_type: Option<String>,

crates/content-telemetry-core/src/services/sessions.rs

@@ -35,16 +35,26 @@ pub async fn create_session(
         );
     }
 
+    // The session-level `data` container and any top-level members this
+    // server does not define are stored as given (spec 5.1.3): nothing
+    // inside either is interpreted, normalised or dropped.
+    let unrecognised = if req.unrecognised_fields.is_empty() {
+        None
+    } else {
+        Some(serde_json::Value::Object(req.unrecognised_fields.clone()))
+    };
+
     sqlx::query_as::<_, SessionRow>(
         r"INSERT INTO sessions (
             organization_id, parent_session_id, initiator_type, initiator,
             content_scope, manifest_ref, conformance_level,
             agent_id, external_session_id, prior_session_ids,
+            session_data, unrecognised_fields,
             user_context, platform_id, client_type, client_info,
             started_at, ended_at
         )
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
-                COALESCE($15, NOW()), $16)
+                $15, $16, COALESCE($17, NOW()), $18)
         RETURNING *",
     )
     .bind(owner_id)
@@ -57,6 +67,8 @@ pub async fn create_session(
     .bind(&req.agent_id)
     .bind(&req.external_session_id)
     .bind(&prior)
+    .bind(&req.data)
+    .bind(&unrecognised)
     .bind(&req.user_context)
     .bind(&req.platform_id)
     .bind(&req.client_type)