diff --git a/rsworkspace/crates/trogon-atlas-cli/src/main.rs b/rsworkspace/crates/trogon-atlas-cli/src/main.rs index b76f20af..73a48288 100644 --- a/rsworkspace/crates/trogon-atlas-cli/src/main.rs +++ b/rsworkspace/crates/trogon-atlas-cli/src/main.rs @@ -73,7 +73,9 @@ enum Cmd { /// after it was read; run apply again to plan from fresh state. Apply { /// Manifest files or directories (directories are walked recursively - /// for .yaml/.yml). + /// for .yaml/.yml), or a board .html file saved by Studio's Export + /// HTML. A board file's external entities are never written; apply + /// fails if the target lacks one. #[arg(short = 'f', long = "filename", required = true, num_args = 1..)] filenames: Vec, /// Run server-side validation for every change without persisting. @@ -88,6 +90,7 @@ enum Cmd { /// Show what apply would change, as a unified diff of canonical proto /// JSON. Exits 1 when differences exist (kubectl diff semantics). Diff { + /// Manifest files, directories or a board .html file, as for apply. #[arg(short = 'f', long = "filename", required = true, num_args = 1..)] filenames: Vec, }, @@ -582,7 +585,7 @@ async fn run( } => { let operation_id = resolve_operation_id(operation_id)?; *operation_id_out = Some(operation_id.clone()); - let documents = manifest::read_paths(&filenames)?; + let files = manifest::read_paths(&filenames)?; let mut client = connect_planning_client( &endpoint, auth_token.as_deref(), @@ -591,7 +594,8 @@ async fn run( dry_run, ) .await?; - let resolved = manifest::load_with_tenant_types(&mut client, documents).await?; + apply::require_external(&mut client, files.external).await?; + let resolved = manifest::load_with_tenant_types(&mut client, files.documents).await?; let plan = apply::plan(&mut client, resolved.manifests).await?; let outcome = apply::apply(&mut client, plan, dry_run, Some(&operation_id)).await?; if text { @@ -625,10 +629,11 @@ async fn run( )) } Cmd::Diff { filenames } => { - let documents = manifest::read_paths(&filenames)?; + let files = manifest::read_paths(&filenames)?; let mut client = connect_client(&endpoint, auth_token.as_deref(), rpc_timeout_secs, branch).await?; - let resolved = manifest::load_with_tenant_types(&mut client, documents).await?; + apply::require_external(&mut client, files.external).await?; + let resolved = manifest::load_with_tenant_types(&mut client, files.documents).await?; let plan = apply::plan(&mut client, resolved.manifests).await?; let (diff, changed) = apply::render_diff(&plan, &resolved.types)?; if text { diff --git a/rsworkspace/crates/trogon-atlas-client/src/apply.rs b/rsworkspace/crates/trogon-atlas-client/src/apply.rs index 7ec1ec00..413af1a2 100644 --- a/rsworkspace/crates/trogon-atlas-client/src/apply.rs +++ b/rsworkspace/crates/trogon-atlas-client/src/apply.rs @@ -21,7 +21,7 @@ use trogon_atlas_proto as pb; use crate::{ client::Client, - manifest::{entity_json_value, LoadedManifest}, + manifest::{entity_json_value, LoadedManifest, ManifestDocument}, precondition::{Revision, StalePlan}, }; @@ -181,6 +181,49 @@ pub async fn plan( .collect() } +/// Refuse a board import whose target lacks a foreign entity the board +/// references. Another namespace owns those entities, so the import never +/// writes them, and only their identity is checked: a bundled spec that no +/// longer transcodes must not hide a missing entity or block a present one. +pub async fn require_external(client: &mut Client, external: Vec) -> Result<()> { + if external.is_empty() { + return Ok(()); + } + let keys = external + .iter() + .map(|d| pb::EntityRef { + kind: d.kind as i32, + id: Some(d.id.clone()), + }) + .collect(); + let found = client + .batch_get_entities(pb::BatchGetEntitiesRequest { keys, dense: true }) + .await + .context("BatchGetEntities failed")? + .into_inner() + .found; + if found.len() != external.len() { + bail!( + "server returned {} found flags for {} keys", + found.len(), + external.len() + ); + } + let missing: Vec = external + .iter() + .zip(found) + .filter(|(_, found)| !found) + .map(|(d, _)| pb::canonical::id_string(d.kind, &d.id)) + .collect(); + if !missing.is_empty() { + crate::invalid_input!( + "the target is missing external entities this board references; apply them from their own namespace first: {}", + missing.join(", ") + ); + } + Ok(()) +} + /// A batch was refused (`BatchMutateResponse.status == FAILED`) for a reason /// other than a stale precondition: a validation rule, a referrer conflict, /// or any other per-op diagnostic in `issues`. The precondition case is diff --git a/rsworkspace/crates/trogon-atlas-client/src/manifest.rs b/rsworkspace/crates/trogon-atlas-client/src/manifest.rs index 63674e4f..20d8e464 100644 --- a/rsworkspace/crates/trogon-atlas-client/src/manifest.rs +++ b/rsworkspace/crates/trogon-atlas-client/src/manifest.rs @@ -125,14 +125,30 @@ pub async fn load_with_tenant_types( Ok(ResolvedManifests { manifests, types }) } +/// Manifest documents read from files, with the external ones a board file +/// bundles kept apart. +#[derive(Debug, Clone, Default)] +pub struct ManifestFiles { + pub documents: Vec, + /// Foreign entities a board references directly. Their own namespace + /// owns them, so an import checks that they exist and never writes them. + pub external: Vec, +} + /// Collect manifest files from files/directories into a sorted, deduped /// list, so callers (apply, fmt) see a stable order regardless of /// filesystem iteration order. Directories are walked recursively for /// `.yaml`/`.yml`. pub(crate) fn collect_yaml_files(paths: &[PathBuf]) -> Result> { + collect_manifest_files(paths, false) +} + +/// Like `collect_yaml_files`, and with `html` also takes a board `.html` +/// file named explicitly. `fmt` passes false: it would rewrite the file as YAML. +fn collect_manifest_files(paths: &[PathBuf], html: bool) -> Result> { let mut files: Vec = Vec::new(); for path in paths { - collect_files(path, &mut files, true)?; + collect_files(path, &mut files, true, html)?; } files.sort(); files.dedup(); @@ -145,18 +161,65 @@ pub(crate) fn collect_yaml_files(paths: &[PathBuf]) -> Result> { /// Collect manifest files from files/directories and parse every document /// in each one. pub fn load_paths(paths: &[PathBuf]) -> Result> { - load_builtin(&read_paths(paths)?) + load_builtin(&read_paths(paths)?.documents) } /// Collect manifest files from files/directories and read every document /// in each one, leaving their specs for `load_with_tenant_types`. -pub fn read_paths(paths: &[PathBuf]) -> Result> { - let files = collect_yaml_files(paths)?; - let mut documents = Vec::new(); - for file in &files { - documents.extend(read_file(file)?); +pub fn read_paths(paths: &[PathBuf]) -> Result { + let mut out = ManifestFiles::default(); + for file in &collect_manifest_files(paths, true)? { + if is_html(file) { + let board = read_board_html(&read_text(file)?, &file.display().to_string())?; + out.documents.extend(board.documents); + out.external.extend(board.external); + } else { + out.documents.extend(read_file(file)?); + } } - Ok(documents) + Ok(out) +} + +fn has_extension(path: &Path, extensions: &[&str]) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| extensions.iter().any(|x| e.eq_ignore_ascii_case(x))) +} + +fn is_html(path: &Path) -> bool { + has_extension(path, &["html"]) +} + +/// Read the manifests a board file saved by Studio's Export HTML carries in +/// its `atlas-snapshot` data block. The block's JSON writes every `<` as +/// `\u003c`, so its first ` Result { + let data = html + .find("id=\"atlas-snapshot\"") + .and_then(|at| html[at..].find('>').map(|end| &html[at + end + 1..])) + .and_then(|rest| rest.find(" Result> { @@ -169,8 +232,9 @@ fn load_builtin(documents: &[ManifestDocument]) -> Result> { /// Walk `path` into `out`. When `explicit` is true (a path the user named /// with `-f`), non-yaml files are errors; when walking a directory they are -/// skipped, matching kubectl's directory walk. -fn collect_files(path: &Path, out: &mut Vec, explicit: bool) -> Result<()> { +/// skipped, matching kubectl's directory walk. A board `.html` file is only +/// taken when named, so a saved board next to its YAML is not read twice. +fn collect_files(path: &Path, out: &mut Vec, explicit: bool, html: bool) -> Result<()> { let meta = std::fs::metadata(path).map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { invalid_input(format!("reading {}: {err}", path.display())) @@ -182,20 +246,17 @@ fn collect_files(path: &Path, out: &mut Vec, explicit: bool) -> Result< for entry in std::fs::read_dir(path).with_context(|| format!("listing {}", path.display()))? { - collect_files(&entry?.path(), out, false)?; + collect_files(&entry?.path(), out, false, html)?; } return Ok(()); } - let is_yaml = path - .extension() - .and_then(|e| e.to_str()) - .is_some_and(|e| e.eq_ignore_ascii_case("yaml") || e.eq_ignore_ascii_case("yml")); - if is_yaml { + if has_extension(path, &["yaml", "yml"]) || (explicit && html && is_html(path)) { out.push(path.to_path_buf()); } else if explicit { crate::invalid_input!( - "{}: not a .yaml/.yml manifest file (unsupported)", - path.display() + "{}: not a .yaml/.yml manifest file{} (unsupported)", + path.display(), + if html { " or .html board file" } else { "" } ); } Ok(()) @@ -207,14 +268,17 @@ pub fn parse_file(path: &Path) -> Result> { } fn read_file(path: &Path) -> Result> { - let text = std::fs::read_to_string(path).map_err(|err| { + read_str(&read_text(path)?, &path.display().to_string()) +} + +fn read_text(path: &Path) -> Result { + std::fs::read_to_string(path).map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { invalid_input(format!("reading {}: {err}", path.display())) } else { anyhow::Error::new(err).context(format!("reading {}", path.display())) } - })?; - read_str(&text, &path.display().to_string()) + }) } /// Parse every YAML document in a string. `source` labels errors. @@ -937,4 +1001,89 @@ spec: { title: A } "error should name the rejected file or reason, got: {msg}" ); } + + /// The data block as Studio's Export HTML writes it: every `<` in the + /// JSON escaped, so a `` inside a doc cannot end the block. + fn board_html(snapshot: &Value) -> String { + let data = serde_json::to_string(snapshot) + .unwrap() + .replace('<', "\\u003c"); + format!( + "\n
\n\ + \n\ + \n\n" + ) + } + + const BOARD_EVENT: &str = "apiVersion: eventmodel.atlas.trogonstack.com/v1alpha1 +kind: Event +metadata: { namespace: acme-checkout, name: order.placed } +spec: { title: 'ends with ' } +"; + + const FOREIGN_EVENT: &str = "apiVersion: eventmodel.atlas.trogonstack.com/v1alpha1 +kind: Event +metadata: { namespace: acme-payments, name: payment.captured } +spec: { title: Payment captured } +"; + + #[test] + fn board_html_reads_the_model_and_keeps_external_entities_apart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("board.html"); + let snapshot = serde_json::json!({ + "version": 1, + "responses": {}, + "manifests": { "model": BOARD_EVENT, "external": FOREIGN_EVENT }, + }); + std::fs::write(&path, board_html(&snapshot)).unwrap(); + + let files = read_paths(&[path]).unwrap(); + let loaded = load_builtin(&files.documents).unwrap(); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].id.slug, "order.placed"); + let Some(pb::entity::Kind::Event(event)) = loaded[0].entity.kind.as_ref() else { + panic!("expected an event"); + }; + assert_eq!(event.title, "ends with "); + let external: Vec<_> = files.external.iter().map(|d| d.id.slug.as_str()).collect(); + assert_eq!(external, ["payment.captured"]); + } + + #[test] + fn board_html_without_manifests_asks_for_a_new_export() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("old.html"); + std::fs::write(&path, board_html(&serde_json::json!({ "responses": {} }))).unwrap(); + let err = read_paths(&[path]).unwrap_err(); + assert!(format!("{err:#}").contains("export it again"), "{err:#}"); + + let page = dir.path().join("page.html"); + std::fs::write(&page, "hello").unwrap(); + let err = read_paths(&[page]).unwrap_err(); + assert!(format!("{err:#}").contains("atlas-snapshot"), "{err:#}"); + } + + #[test] + fn html_is_read_only_when_named_and_never_formatted() { + let dir = tempfile::tempdir().unwrap(); + let html = dir.path().join("board.html"); + let snapshot = serde_json::json!({ "manifests": { "model": BOARD_EVENT, "external": "" } }); + std::fs::write(&html, board_html(&snapshot)).unwrap(); + std::fs::write(dir.path().join("payments.yaml"), FOREIGN_EVENT).unwrap(); + + // A directory walk keeps to YAML, so a saved board next to manifests is not applied twice. + let walked = read_paths(&[dir.path().to_path_buf()]).unwrap(); + let slugs: Vec<_> = walked + .documents + .iter() + .map(|d| d.id.slug.as_str()) + .collect(); + assert_eq!(slugs, ["payment.captured"]); + assert!(read_paths(std::slice::from_ref(&html)) + .unwrap() + .external + .is_empty()); + assert!(collect_yaml_files(&[html]).is_err()); + } } diff --git a/rsworkspace/crates/trogon-atlas-client/tests/export_integration.rs b/rsworkspace/crates/trogon-atlas-client/tests/export_integration.rs index 8e3ae48f..b8474be5 100644 --- a/rsworkspace/crates/trogon-atlas-client/tests/export_integration.rs +++ b/rsworkspace/crates/trogon-atlas-client/tests/export_integration.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use serde::Deserialize as _; use tokio::net::TcpListener; -use trogon_atlas_client::{client, export, manifest}; +use trogon_atlas_client::{apply, client, export, manifest}; use trogon_atlas_proto as pb; use trogon_atlas_proto::event_model_service_server::EventModelServiceServer; use trogon_atlas_server::{metrics::ServerMetrics, service::EventModelServiceImpl}; @@ -956,6 +956,100 @@ async fn board_manifests_carry_the_board_and_load_back_without_drift() { assert_eq!(copied.external_manifests, out.external_manifests); } +/// Write `model` and `external` into a board file the way Studio's Export +/// HTML does, `<` escaped so the YAML cannot end the data block. +fn write_board_html(dir: &std::path::Path, model: &str, external: &str) -> std::path::PathBuf { + let snapshot = serde_json::json!({ "manifests": { "model": model, "external": external } }); + let data = snapshot.to_string().replace('<', "\\u003c"); + let path = dir.join("shop.html"); + std::fs::write( + &path, + format!(""), + ) + .unwrap(); + path +} + +#[tokio::test] +async fn board_html_applies_into_a_fresh_namespace_and_checks_external_entities() { + let endpoint = start_server().await; + let mut client = client::connect(&endpoint, None, 30).await.unwrap(); + seed(&mut client, BOARD_PAYMENTS).await; + seed(&mut client, BOARD_SHOP).await; + let out = export_board(&mut client, "board-src").await.unwrap(); + let dir = tempfile::tempdir().unwrap(); + + let model = out.manifests.replace("board-src", "board-html"); + let files = manifest::read_paths(&[write_board_html( + dir.path(), + &model, + &out.external_manifests, + )]) + .unwrap(); + apply::require_external(&mut client, files.external) + .await + .unwrap(); + let resolved = manifest::load_with_tenant_types(&mut client, files.documents.clone()) + .await + .unwrap(); + let plan = apply::plan(&mut client, resolved.manifests).await.unwrap(); + assert!(plan.iter().all(|c| c.action == apply::Action::Create)); + apply::apply(&mut client, plan, false, None).await.unwrap(); + let resolved = manifest::load_with_tenant_types(&mut client, files.documents) + .await + .unwrap(); + for change in apply::plan(&mut client, resolved.manifests).await.unwrap() { + assert_eq!( + change.action, + apply::Action::Unchanged, + "{}", + change.manifest.source + ); + } + + // A target that lacks a foreign entity the board uses is refused before any write. + let gone = |yaml: &str| yaml.replace("board-pay", "board-gone"); + let path = write_board_html(dir.path(), &gone(&model), &gone(&out.external_manifests)); + let files = manifest::read_paths(&[path]).unwrap(); + let err = apply::require_external(&mut client, files.external) + .await + .unwrap_err(); + assert!( + format!("{err:#}").contains("event:board-gone/payment.captured"), + "{err:#}" + ); + + // The bundled copy is never written, so a spec this client cannot transcode + // must neither hide a missing entity nor block a present one. + let untyped = |ns: &str| { + format!( + "apiVersion: eventmodel.atlas.trogonstack.com/v1alpha1 +kind: Event +metadata: {{ namespace: {ns}, name: payment.captured }} +spec: + title: Payment captured + metadata: + - '@type': type.googleapis.com/acme.payments.v1.Retired + note: gone +" + ) + }; + let path = write_board_html(dir.path(), &model, &untyped("board-gone")); + let files = manifest::read_paths(&[path]).unwrap(); + let err = apply::require_external(&mut client, files.external) + .await + .unwrap_err(); + assert!( + format!("{err:#}").contains("missing external entities"), + "{err:#}" + ); + let path = write_board_html(dir.path(), &model, &untyped("board-pay")); + let files = manifest::read_paths(&[path]).unwrap(); + apply::require_external(&mut client, files.external) + .await + .unwrap(); +} + #[tokio::test] async fn board_manifests_for_a_missing_event_model_are_not_found() { let endpoint = start_server().await;