diff --git a/README.md b/README.md index da1f80d..9122acc 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,31 @@ making it ideal for building views server-side and shipping them to Python, WASM ## Regenerating the metamodel -In order to regenerate the metamodel: - -* Make sure you have a python virtual env with linkml_runtime (python!) installed, and that its active -* In the `../linkml` folder there should be a linkml checkout that is on a branch with the rust generator -* run the `regen.sh` script from the root of this repo - -Note that now the metamodel is generated from src/schemaview/tests/data/meta.yaml. - -### TODOs - -* generate the metamodel directly from the linkml meta repository +`src/metamodel` (the `linkml_meta` crate) is generated and must never be edited by hand. +The generator is `gen-rust`, which lives in a [linkml](https://github.com/linkml/linkml) +checkout rather than in this repo; the input schema is `src/schemaview/tests/data/meta.yaml`. + +* Put a linkml checkout at `../linkml` on a branch carrying the rust generator, or point + `LINKML_DIR` at one. Recreate its venv with `cd ../linkml && uv sync` if it is missing. +* Regenerate in place: `./regen.sh` +* Check reproducibility: `./regen.sh --check` — regenerates into a temp dir, formats it the + same way, and diffs against the committed crate without writing anything. + +An empty `--check` diff means the committed crate is exactly what that generator emits. A +non-empty one means either the checkout is on a different revision than the crate was built +from, or something was hand-edited into generated code — and a regen would silently revert +it. That has happened before (a per-key map merge strategy), which is why the check exists. + +### Keeping regeneration reproducible + +`./regen.sh` stamps `src/metamodel/GENERATED_FROM` with the generator revision it used and +with any generator commits that are not in `linkml/main` yet. While that list is non-empty +the crate cannot be reproduced from upstream linkml alone, so a generator fix belongs +upstream as a linkml PR *before* the regenerated crate is committed here. + +Each open linkml generator PR this crate is waiting on has its own tracking issue, so they close +as they merge; [#109](https://github.com/Kapernikov/rust-linkml-core/issues/109) is the +regeneration itself and links to them. ## Development on the Python bindings diff --git a/regen.sh b/regen.sh old mode 100644 new mode 100755 index e473ec5..849428e --- a/regen.sh +++ b/regen.sh @@ -1,14 +1,109 @@ #!/bin/bash +# Regenerate the `linkml_meta` crate from meta.yaml with the linkml rust +# generator, and record which generator revision produced it. +# +# ./regen.sh regenerate src/metamodel/ in place +# ./regen.sh --check regenerate into a temp dir and diff; no writes +# +# `--check` is the reproducibility gate: the committed crate must be exactly +# what the generator in $LINKML_DIR emits. A non-empty diff means either the +# checkout moved or something was hand-edited into generated code. Both are +# worth knowing about before a regen silently reverts one of them. +# +# The generator lives in a linkml checkout, not in this repo. Point at it with +# LINKML_DIR (default `../linkml`) on a branch carrying the rust generator, and +# recreate its venv with `cd $LINKML_DIR && uv sync` if it is missing. set -euo pipefail -# Run the generator from the linkml checkout via its uv-managed venv. -# Recreate the venv with `cd ../linkml && uv sync` if it is missing. -cd ../linkml -uv run gen-rust ../rust-linkml-core/src/schemaview/tests/data/meta.yaml \ - --output ../rust-linkml-core/src/metamodel/ \ - --force --serde -n linkml_meta --stacktrace +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# The default sibling checkout is a sibling of the *main* working tree, so this +# still finds it when run from a git worktree nested inside the repo. +git_common=$(cd "$REPO_ROOT" && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true) +if [[ -n $git_common && -d $git_common ]]; then + main_checkout=$(cd "$git_common/.." && pwd) +else + main_checkout=$REPO_ROOT +fi +LINKML_DIR=${LINKML_DIR:-$main_checkout/../linkml} +META_YAML=$REPO_ROOT/src/schemaview/tests/data/meta.yaml +CRATE_DIR=$REPO_ROOT/src/metamodel +PROVENANCE=$CRATE_DIR/GENERATED_FROM -# Reformat the Rust workspace to clean up generated code -cd ../rust-linkml-core +check=0 +if [[ ${1:-} == "--check" ]]; then + check=1 +elif [[ $# -gt 0 ]]; then + echo "usage: $0 [--check]" >&2 + exit 2 +fi + +if [[ ! -d $LINKML_DIR ]]; then + echo "no linkml checkout at $LINKML_DIR (override with LINKML_DIR=...)" >&2 + exit 1 +fi +LINKML_DIR=$(cd "$LINKML_DIR" && pwd) + +# The revision that produced this output, and — more useful when the output +# does not match — which generator commits are not upstream yet. Anything +# listed here has to be merged and this file re-stamped before another machine +# can reproduce the crate. +revision=$(git -C "$LINKML_DIR" rev-parse HEAD) +described=$(git -C "$LINKML_DIR" describe --always --dirty) +unmerged=$(git -C "$LINKML_DIR" log --oneline linkml/main..HEAD 2>/dev/null || true) + +out_dir=$CRATE_DIR +tmp_dir= +if [[ $check -eq 1 ]]; then + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + out_dir=$tmp_dir +fi + +echo "Generating linkml_meta from $META_YAML" +echo " generator: $LINKML_DIR @ $described" +(cd "$LINKML_DIR" && uv run gen-rust "$META_YAML" \ + --output "$out_dir" \ + --force --serde -n linkml_meta --stacktrace) + +if [[ $check -eq 1 ]]; then + # The committed crate is formatted; format the fresh output the same way + # before comparing, or every line looks changed. + find "$out_dir" -name '*.rs' -print0 | xargs -0 rustfmt --edition 2021 + echo "Diffing against $CRATE_DIR ..." + # Only the generated sources and Cargo.toml are compared; a --check run + # never touches GENERATED_FROM, so exclude it. `*~` excludes the editor + # backup that is committed next to Cargo.toml. + if diff -r --exclude=GENERATED_FROM --exclude=target --exclude='*~' \ + "$CRATE_DIR" "$out_dir"; then + echo "OK: committed crate reproduces from $described" + else + echo + echo "DRIFT: the committed crate is not what this generator emits." >&2 + echo "Either \$LINKML_DIR is on a different revision than GENERATED_FROM," >&2 + echo "or generated code was edited by hand." >&2 + exit 1 + fi + exit 0 +fi + +{ + echo "# Provenance of the generated linkml_meta crate. Written by regen.sh." + echo "# Do not edit by hand." + echo "generator_repo = $(git -C "$LINKML_DIR" remote get-url origin 2>/dev/null || echo unknown)" + echo "generator_revision = $revision" + echo "generator_describe = $described" + echo "meta_yaml = src/schemaview/tests/data/meta.yaml" + echo + if [[ -n $unmerged ]]; then + echo "# Generator commits NOT in linkml/main. Until these are merged, this" + echo "# crate cannot be reproduced from upstream linkml alone." + while IFS= read -r line; do echo "# $line"; done <<<"$unmerged" + else + echo "# Every generator commit is in linkml/main; reproducible from upstream." + fi +} >"$PROVENANCE" + +echo "Wrote $PROVENANCE" echo "Running cargo fmt over the workspace..." -cargo fmt --all +(cd "$REPO_ROOT" && cargo fmt --all) diff --git a/src/runtime/tests/data/custom_type_data.yaml b/src/runtime/tests/data/custom_type_data.yaml index a11df48..2563f9f 100644 --- a/src/runtime/tests/data/custom_type_data.yaml +++ b/src/runtime/tests/data/custom_type_data.yaml @@ -2,3 +2,5 @@ name: Brussels location: "POINT(4.3517 50.8503)" homepage: "https://www.brussels.be" related_doc: "https://example.com/docs/brussels" +mirror: "https://mirror.brussels.be" +track_length: 1200 diff --git a/src/runtime/tests/data/custom_type_schema.yaml b/src/runtime/tests/data/custom_type_schema.yaml index 0834309..4a4853c 100644 --- a/src/runtime/tests/data/custom_type_schema.yaml +++ b/src/runtime/tests/data/custom_type_schema.yaml @@ -13,6 +13,15 @@ types: typeof: string title: Well-known Text Literal description: A Well-known Text serialization of a Geometry object. + # Derived types carrying no `uri` of their own: everything they contribute to + # serialisation is inherited through `typeof`. + weblink: + typeof: uri + base: URI + repr: str + trackLength: + typeof: integer + base: int classes: Place: attributes: @@ -24,3 +33,7 @@ classes: range: uri related_doc: range: uriorcurie + mirror: + range: weblink + track_length: + range: trackLength diff --git a/src/runtime/tests/turtle_custom_types.rs b/src/runtime/tests/turtle_custom_types.rs index 2e80d08..85df34a 100644 --- a/src/runtime/tests/turtle_custom_types.rs +++ b/src/runtime/tests/turtle_custom_types.rs @@ -132,6 +132,70 @@ fn turtle_uri_range_emits_named_node() { ); } +/// A type derived from `uri` with no `uri` of its own is still an IRI, so its +/// values are named nodes. This is the user-visible half of issue #108: while +/// `typeof:` is dropped at deserialization the derived type inherits nothing and +/// this serializes as a plain literal instead. Expected to pass unchanged once +/// linkml/linkml#3919 lands and the metamodel is regenerated (see #109). +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn turtle_type_derived_from_uri_emits_named_node() { + let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml"); + let class = sv + .get_class(&Identifier::new("Place"), &conv) + .unwrap() + .unwrap(); + let v = load_yaml_file( + Path::new(&data_path("custom_type_data.yaml")), + &sv, + &class, + &conv, + ) + .unwrap() + .into_instance() + .unwrap(); + let ttl = turtle_to_string(&v, &sv, &schema, &conv, TurtleOptions { skolem: false }).unwrap(); + + assert!( + ttl.contains(""), + "a `typeof: uri` range should emit a named node. Got:\n{}", + ttl + ); + assert!( + !ttl.contains("\"https://mirror.brussels.be\""), + "a `typeof: uri` range should NOT be a string literal. Got:\n{}", + ttl + ); +} + +/// Same inheritance, on the datatype rather than the IRI disposition: a type +/// derived from `integer` carries `xsd:integer`. See #108. +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn turtle_type_derived_from_integer_carries_its_parents_datatype() { + let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml"); + let class = sv + .get_class(&Identifier::new("Place"), &conv) + .unwrap() + .unwrap(); + let v = load_yaml_file( + Path::new(&data_path("custom_type_data.yaml")), + &sv, + &class, + &conv, + ) + .unwrap() + .into_instance() + .unwrap(); + let ttl = turtle_to_string(&v, &sv, &schema, &conv, TurtleOptions { skolem: false }).unwrap(); + + assert!( + !ttl.contains("\"1200\" ") && !ttl.contains("\"1200\";") && !ttl.contains("\"1200\"."), + "a `typeof: integer` range should not serialize as a plain literal. Got:\n{}", + ttl + ); +} + #[test] fn turtle_uriorcurie_range_emits_named_node() { let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml"); diff --git a/src/schemaview/tests/data/typeof_schema.yaml b/src/schemaview/tests/data/typeof_schema.yaml new file mode 100644 index 0000000..a787c38 --- /dev/null +++ b/src/schemaview/tests/data/typeof_schema.yaml @@ -0,0 +1,37 @@ +id: https://example.com/typeofs +name: typeofs +prefixes: + typeofs: https://example.com/typeofs/ + xsd: http://www.w3.org/2001/XMLSchema# + linkml: https://w3id.org/linkml/ +default_prefix: typeofs +imports: +- linkml:types +types: + # Derived types spelled the way LinkML spells them: `typeof`, not `typeof_`. + trackLength: + typeof: integer + base: int + weblink: + typeof: uri + base: URI + repr: str + # Two levels down, to check the walk does not stop at the first parent. + mainTrackLength: + typeof: trackLength + base: int +classes: + Base: + abstract: true + attributes: + name: + range: string + Thing: + is_a: Base + attributes: + length: + range: trackLength + main_length: + range: mainTrackLength + homepage: + range: weblink diff --git a/src/schemaview/tests/typeof_inheritance.rs b/src/schemaview/tests/typeof_inheritance.rs new file mode 100644 index 0000000..60d747c --- /dev/null +++ b/src/schemaview/tests/typeof_inheritance.rs @@ -0,0 +1,134 @@ +//! LinkML spells the derived-type key `typeof` and the class flag `abstract`. +//! Both are Rust keywords, so the generated metamodel fields are `typeof_` and +//! `abstract_`; without a serde rename the key on the wire never matches the +//! field and the value is silently dropped, leaving derived types with no +//! parent at all. +//! +//! The metamodel is generated, so the fix is a generator change (linkml/linkml#3919) +//! followed by a regen here. Until that lands these are `#[ignore]`d rather than +//! deleted: they are the executable record of issue #108, and are expected to pass +//! unchanged once the regenerated metamodel is committed. Un-ignore them in that +//! commit; see #109 for the regeneration gate. + +use linkml_schemaview::identifier::{converter_from_schemas, Identifier}; +use linkml_schemaview::io::from_yaml; +use linkml_schemaview::schemaview::SchemaView; +use std::path::{Path, PathBuf}; + +fn data_path(name: &str) -> PathBuf { + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("data"); + p.push(name); + p +} + +fn load() -> (SchemaView, linkml_schemaview::Converter) { + let schema = from_yaml(Path::new(&data_path("typeof_schema.yaml"))).unwrap(); + let types_schema = from_yaml(Path::new(&data_path("types.yaml"))).unwrap(); + let mut sv = SchemaView::new(); + sv.add_schema(schema.clone()).unwrap(); + sv.add_schema_with_import_ref( + types_schema.clone(), + Some((schema.id.clone(), "linkml:types".to_string())), + ) + .unwrap(); + let conv = converter_from_schemas([&schema, &types_schema]); + (sv, conv) +} + +fn range_info( + sv: &SchemaView, + conv: &linkml_schemaview::Converter, + class_name: &str, + slot_name: &str, +) -> linkml_schemaview::slotview::RangeInfo { + let class = sv + .get_class(&Identifier::new(class_name), conv) + .unwrap() + .unwrap(); + class + .slot(&Identifier::Name(slot_name.to_string())) + .unwrap_or_else(|| panic!("slot '{}' not found on '{}'", slot_name, class_name)) + .get_range_info() + .first() + .cloned() + .unwrap() +} + +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn typeof_survives_deserialization() { + let schema = from_yaml(Path::new(&data_path("typeof_schema.yaml"))).unwrap(); + let types = schema.types.as_ref().expect("schema declares types"); + assert_eq!( + types.get("trackLength").unwrap().typeof_.as_deref(), + Some("integer"), + "`typeof:` must not be dropped on the way in" + ); +} + +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn abstract_survives_deserialization() { + let schema = from_yaml(Path::new(&data_path("typeof_schema.yaml"))).unwrap(); + let classes = schema.classes.as_ref().expect("schema declares classes"); + assert_eq!( + classes.get("Base").unwrap().abstract_, + Some(true), + "`abstract:` must not be dropped on the way in" + ); +} + +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn type_ancestors_walks_the_whole_typeof_chain() { + let (sv, conv) = load(); + let ancestors = sv + .type_ancestors(&Identifier::new("mainTrackLength"), &conv) + .unwrap(); + let names: Vec = ancestors.iter().map(|a| a.to_string()).collect(); + assert_eq!( + names, + vec![ + "mainTrackLength".to_string(), + "trackLength".to_string(), + "integer".to_string() + ], + "a derived type inherits through every level of the chain" + ); +} + +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn derived_type_inherits_its_parents_datatype() { + let (sv, conv) = load(); + for slot in ["length", "main_length"] { + let ri = range_info(&sv, &conv, "Thing", slot); + assert_eq!( + ri.rdf_datatype_iri.as_deref(), + Some("http://www.w3.org/2001/XMLSchema#integer"), + "'{}' should inherit xsd:integer through its typeof chain", + slot + ); + assert!( + ri.is_integer(), + "'{}' should canonicalise as an integer", + slot + ); + } +} + +/// The case with user-visible serialisation impact: a type derived from `uri` +/// is an IRI, so its values are named nodes rather than literals. +#[test] +#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"] +fn type_derived_from_uri_is_an_iri_range() { + let (sv, conv) = load(); + let ri = range_info(&sv, &conv, "Thing", "homepage"); + assert!( + ri.is_range_iri, + "a `typeof: uri` range must be flagged as an IRI" + ); + assert_eq!(ri.rdf_datatype_iri, None); +}