chore(metamodel): make regeneration reproducible, and pin down the dropped-typeof bug

README.md

@@ -31,17 +31,31 @@ making it ideal for building views server-side and shipping them to Python, WASM
 
 ## Regenerating the metamodel
 
-In order to regenerate the metamodel:
-
-* Make sure you have a python virtual env with linkml_runtime (python!) installed, and that its active
-* In the `../linkml` folder there should be a linkml checkout that is on a branch with the rust generator
-* run the `regen.sh` script from the root of this repo
-
-Note that now the metamodel is generated from src/schemaview/tests/data/meta.yaml.
-
-### TODOs
-
-* generate the metamodel directly from the linkml meta repository
+`src/metamodel` (the `linkml_meta` crate) is generated and must never be edited by hand.
+The generator is `gen-rust`, which lives in a [linkml](https://github.com/linkml/linkml)
+checkout rather than in this repo; the input schema is `src/schemaview/tests/data/meta.yaml`.
+
+* Put a linkml checkout at `../linkml` on a branch carrying the rust generator, or point
+  `LINKML_DIR` at one. Recreate its venv with `cd ../linkml && uv sync` if it is missing.
+* Regenerate in place: `./regen.sh`
+* Check reproducibility: `./regen.sh --check` — regenerates into a temp dir, formats it the
+  same way, and diffs against the committed crate without writing anything.
+
+An empty `--check` diff means the committed crate is exactly what that generator emits. A
+non-empty one means either the checkout is on a different revision than the crate was built
+from, or something was hand-edited into generated code — and a regen would silently revert
+it. That has happened before (a per-key map merge strategy), which is why the check exists.
+
+### Keeping regeneration reproducible
+
+`./regen.sh` stamps `src/metamodel/GENERATED_FROM` with the generator revision it used and
+with any generator commits that are not in `linkml/main` yet. While that list is non-empty
+the crate cannot be reproduced from upstream linkml alone, so a generator fix belongs
+upstream as a linkml PR *before* the regenerated crate is committed here.
+
+Each open linkml generator PR this crate is waiting on has its own tracking issue, so they close
+as they merge; [#109](https://github.com/Kapernikov/rust-linkml-core/issues/109) is the
+regeneration itself and links to them.
 
 ## Development on the Python bindings
 

regen.sh

@@ -1,14 +1,109 @@
 #!/bin/bash
+# Regenerate the `linkml_meta` crate from meta.yaml with the linkml rust
+# generator, and record which generator revision produced it.
+#
+#   ./regen.sh            regenerate src/metamodel/ in place
+#   ./regen.sh --check     regenerate into a temp dir and diff; no writes
+#
+# `--check` is the reproducibility gate: the committed crate must be exactly
+# what the generator in $LINKML_DIR emits. A non-empty diff means either the
+# checkout moved or something was hand-edited into generated code. Both are
+# worth knowing about before a regen silently reverts one of them.
+#
+# The generator lives in a linkml checkout, not in this repo. Point at it with
+# LINKML_DIR (default `../linkml`) on a branch carrying the rust generator, and
+# recreate its venv with `cd $LINKML_DIR && uv sync` if it is missing.
 set -euo pipefail
 
-# Run the generator from the linkml checkout via its uv-managed venv.
-# Recreate the venv with `cd ../linkml && uv sync` if it is missing.
-cd ../linkml
-uv run gen-rust ../rust-linkml-core/src/schemaview/tests/data/meta.yaml \
-  --output ../rust-linkml-core/src/metamodel/ \
-  --force --serde -n linkml_meta --stacktrace
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# The default sibling checkout is a sibling of the *main* working tree, so this
+# still finds it when run from a git worktree nested inside the repo.
+git_common=$(cd "$REPO_ROOT" && git rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)
+if [[ -n $git_common && -d $git_common ]]; then
+  main_checkout=$(cd "$git_common/.." && pwd)
+else
+  main_checkout=$REPO_ROOT
+fi
+LINKML_DIR=${LINKML_DIR:-$main_checkout/../linkml}
+META_YAML=$REPO_ROOT/src/schemaview/tests/data/meta.yaml
+CRATE_DIR=$REPO_ROOT/src/metamodel
+PROVENANCE=$CRATE_DIR/GENERATED_FROM
 
-# Reformat the Rust workspace to clean up generated code
-cd ../rust-linkml-core
+check=0
+if [[ ${1:-} == "--check" ]]; then
+  check=1
+elif [[ $# -gt 0 ]]; then
+  echo "usage: $0 [--check]" >&2
+  exit 2
+fi
+
+if [[ ! -d $LINKML_DIR ]]; then
+  echo "no linkml checkout at $LINKML_DIR (override with LINKML_DIR=...)" >&2
+  exit 1
+fi
+LINKML_DIR=$(cd "$LINKML_DIR" && pwd)
+
+# The revision that produced this output, and — more useful when the output
+# does not match — which generator commits are not upstream yet. Anything
+# listed here has to be merged and this file re-stamped before another machine
+# can reproduce the crate.
+revision=$(git -C "$LINKML_DIR" rev-parse HEAD)
+described=$(git -C "$LINKML_DIR" describe --always --dirty)
+unmerged=$(git -C "$LINKML_DIR" log --oneline linkml/main..HEAD 2>/dev/null || true)
+
+out_dir=$CRATE_DIR
+tmp_dir=
+if [[ $check -eq 1 ]]; then
+  tmp_dir=$(mktemp -d)
+  # shellcheck disable=SC2064
+  trap "rm -rf '$tmp_dir'" EXIT
+  out_dir=$tmp_dir
+fi
+
+echo "Generating linkml_meta from $META_YAML"
+echo "  generator: $LINKML_DIR @ $described"
+(cd "$LINKML_DIR" && uv run gen-rust "$META_YAML" \
+  --output "$out_dir" \
+  --force --serde -n linkml_meta --stacktrace)
+
+if [[ $check -eq 1 ]]; then
+  # The committed crate is formatted; format the fresh output the same way
+  # before comparing, or every line looks changed.
+  find "$out_dir" -name '*.rs' -print0 | xargs -0 rustfmt --edition 2021
+  echo "Diffing against $CRATE_DIR ..."
+  # Only the generated sources and Cargo.toml are compared; a --check run
+  # never touches GENERATED_FROM, so exclude it. `*~` excludes the editor
+  # backup that is committed next to Cargo.toml.
+  if diff -r --exclude=GENERATED_FROM --exclude=target --exclude='*~' \
+    "$CRATE_DIR" "$out_dir"; then
+    echo "OK: committed crate reproduces from $described"
+  else
+    echo
+    echo "DRIFT: the committed crate is not what this generator emits." >&2
+    echo "Either \$LINKML_DIR is on a different revision than GENERATED_FROM," >&2
+    echo "or generated code was edited by hand." >&2
+    exit 1
+  fi
+  exit 0
+fi
+
+{
+  echo "# Provenance of the generated linkml_meta crate. Written by regen.sh."
+  echo "# Do not edit by hand."
+  echo "generator_repo = $(git -C "$LINKML_DIR" remote get-url origin 2>/dev/null || echo unknown)"
+  echo "generator_revision = $revision"
+  echo "generator_describe = $described"
+  echo "meta_yaml = src/schemaview/tests/data/meta.yaml"
+  echo
+  if [[ -n $unmerged ]]; then
+    echo "# Generator commits NOT in linkml/main. Until these are merged, this"
+    echo "# crate cannot be reproduced from upstream linkml alone."
+    while IFS= read -r line; do echo "# $line"; done <<<"$unmerged"
+  else
+    echo "# Every generator commit is in linkml/main; reproducible from upstream."
+  fi
+} >"$PROVENANCE"
+
+echo "Wrote $PROVENANCE"
 echo "Running cargo fmt over the workspace..."
-cargo fmt --all
+(cd "$REPO_ROOT" && cargo fmt --all)

src/runtime/tests/data/custom_type_data.yaml

@@ -2,3 +2,5 @@ name: Brussels
 location: "POINT(4.3517 50.8503)"
 homepage: "https://www.brussels.be"
 related_doc: "https://example.com/docs/brussels"
+mirror: "https://mirror.brussels.be"
+track_length: 1200

src/runtime/tests/data/custom_type_schema.yaml

@@ -13,6 +13,15 @@ types:
     typeof: string
     title: Well-known Text Literal
     description: A Well-known Text serialization of a Geometry object.
+  # Derived types carrying no `uri` of their own: everything they contribute to
+  # serialisation is inherited through `typeof`.
+  weblink:
+    typeof: uri
+    base: URI
+    repr: str
+  trackLength:
+    typeof: integer
+    base: int
 classes:
   Place:
     attributes:
@@ -24,3 +33,7 @@ classes:
         range: uri
       related_doc:
         range: uriorcurie
+      mirror:
+        range: weblink
+      track_length:
+        range: trackLength

src/runtime/tests/turtle_custom_types.rs

@@ -132,6 +132,70 @@ fn turtle_uri_range_emits_named_node() {
     );
 }
 
+/// A type derived from `uri` with no `uri` of its own is still an IRI, so its
+/// values are named nodes. This is the user-visible half of issue #108: while
+/// `typeof:` is dropped at deserialization the derived type inherits nothing and
+/// this serializes as a plain literal instead. Expected to pass unchanged once
+/// linkml/linkml#3919 lands and the metamodel is regenerated (see #109).
+#[test]
+#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"]
+fn turtle_type_derived_from_uri_emits_named_node() {
+    let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml");
+    let class = sv
+        .get_class(&Identifier::new("Place"), &conv)
+        .unwrap()
+        .unwrap();
+    let v = load_yaml_file(
+        Path::new(&data_path("custom_type_data.yaml")),
+        &sv,
+        &class,
+        &conv,
+    )
+    .unwrap()
+    .into_instance()
+    .unwrap();
+    let ttl = turtle_to_string(&v, &sv, &schema, &conv, TurtleOptions { skolem: false }).unwrap();
+
+    assert!(
+        ttl.contains("<https://mirror.brussels.be>"),
+        "a `typeof: uri` range should emit a named node. Got:\n{}",
+        ttl
+    );
+    assert!(
+        !ttl.contains("\"https://mirror.brussels.be\""),
+        "a `typeof: uri` range should NOT be a string literal. Got:\n{}",
+        ttl
+    );
+}
+
+/// Same inheritance, on the datatype rather than the IRI disposition: a type
+/// derived from `integer` carries `xsd:integer`. See #108.
+#[test]
+#[ignore = "blocked on linkml/linkml#3919 + a metamodel regen; see issue #108"]
+fn turtle_type_derived_from_integer_carries_its_parents_datatype() {
+    let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml");
+    let class = sv
+        .get_class(&Identifier::new("Place"), &conv)
+        .unwrap()
+        .unwrap();
+    let v = load_yaml_file(
+        Path::new(&data_path("custom_type_data.yaml")),
+        &sv,
+        &class,
+        &conv,
+    )
+    .unwrap()
+    .into_instance()
+    .unwrap();
+    let ttl = turtle_to_string(&v, &sv, &schema, &conv, TurtleOptions { skolem: false }).unwrap();
+
+    assert!(
+        !ttl.contains("\"1200\" ") && !ttl.contains("\"1200\";") && !ttl.contains("\"1200\"."),
+        "a `typeof: integer` range should not serialize as a plain literal. Got:\n{}",
+        ttl
+    );
+}
+
 #[test]
 fn turtle_uriorcurie_range_emits_named_node() {
     let (schema, sv, conv) = load_schema_with_types("custom_type_schema.yaml");

src/schemaview/tests/data/typeof_schema.yaml

@@ -0,0 +1,37 @@
+id: https://example.com/typeofs
+name: typeofs
+prefixes:
+  typeofs: https://example.com/typeofs/
+  xsd: http://www.w3.org/2001/XMLSchema#
+  linkml: https://w3id.org/linkml/
+default_prefix: typeofs
+imports:
+- linkml:types
+types:
+  # Derived types spelled the way LinkML spells them: `typeof`, not `typeof_`.
+  trackLength:
+    typeof: integer
+    base: int
+  weblink:
+    typeof: uri
+    base: URI
+    repr: str
+  # Two levels down, to check the walk does not stop at the first parent.
+  mainTrackLength:
+    typeof: trackLength
+    base: int
+classes:
+  Base:
+    abstract: true
+    attributes:
+      name:
+        range: string
+  Thing:
+    is_a: Base
+    attributes:
+      length:
+        range: trackLength
+      main_length:
+        range: mainTrackLength
+      homepage:
+        range: weblink