From d95bb971df909042576611c7813231fbc3ebdde6 Mon Sep 17 00:00:00 2001 From: Fritz Kunstler <36011164+dfkunstler@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:11:23 -0500 Subject: [PATCH] fix: check the manifest format version before the JSON Schema NAM-022 says a consumer reads `format` before it applies the manifest JSON Schema, and rejects an unsupported value with a statement that the format version is not supported. NAM-021 says a manifest that is not a JSON object, or has no `format` member, is invalid. validate_manifest now checks these three cases first and stops with one failure, so a format 2 manifest reports "format 2 is not supported", not a JSON Schema failure. The format check after the JSON Schema is gone, because it could no longer fail. Closes #57 --- .agents/summary/workflows.md | 5 ++-- AGENTS.md | 2 +- DEVELOPING.md | 4 ++-- scripts/manifest.py | 37 ++++++++++++++++++++--------- tests/test_manifest.py | 46 +++++++++++++++++++++++++++++------- 5 files changed, 69 insertions(+), 25 deletions(-) diff --git a/.agents/summary/workflows.md b/.agents/summary/workflows.md index a00f992..6077d6f 100644 --- a/.agents/summary/workflows.md +++ b/.agents/summary/workflows.md @@ -22,8 +22,9 @@ flowchart TD ### 1a. Hoist (`scripts/hoist_names.py`) -1. `load_validated_manifest` (`scripts/manifest.py`) checks `names.json` in order: the manifest JSON Schema of the same - artifact (`jsonschema-rs`), the format version and OSCAL release, `schema.sha256` against the artifact schema bytes, +1. `load_validated_manifest` (`scripts/manifest.py`) checks `names.json` in order: the format version (a JSON object with + a supported `format`; a failure stops the checks, NAM-022), the manifest JSON Schema of the same artifact + (`jsonschema-rs`), the OSCAL release, `schema.sha256` against the artifact schema bytes, then unique names, pointers that occur once and resolve, and type references that name an entry. It raises `ManifestError` with every failure. The hoist reads the schema bytes from `paths.schema` one time and parses the same bytes that the check hashed. diff --git a/AGENTS.md b/AGENTS.md index ee3f865..fc4fe7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,7 @@ Neither `v1/__init__.py` nor the top-level `__init__.py` defines symbols — bot Names come from the store's naming manifest. The binding holds no naming rule for a manifest type. `hatch run generate` runs three steps: -1. **Hoist.** `python scripts/hoist_names.py` fetches the spec artifact pinned in `scripts/spec_artifact.py` (`0.8.0+oscal.1.2.3`, sha256-checked, cached in `tests/.corpus_cache/`), validates its `names.json` with `scripts/manifest.py` (manifest JSON Schema via `jsonschema-rs`, format and release, artifact-schema sha256, unique names, resolvable pointers), reads the complete OSCAL schema of the same artifact one time (the manifest check hashes the bytes that the hoist parses), and writes `build/codegen/oscal_complete_schema.json` (git-ignored). In that copy each manifest site is `definitions/` with a `$ref` at the site. A merge with different bodies, or a closed-enum value that its base datatype rejects, fails the run. +1. **Hoist.** `python scripts/hoist_names.py` fetches the spec artifact pinned in `scripts/spec_artifact.py` (`0.8.0+oscal.1.2.3`, sha256-checked, cached in `tests/.corpus_cache/`), validates its `names.json` with `scripts/manifest.py` (format version first, then manifest JSON Schema via `jsonschema-rs`, release, artifact-schema sha256, unique names, resolvable pointers), reads the complete OSCAL schema of the same artifact one time (the manifest check hashes the bytes that the hoist parses), and writes `build/codegen/oscal_complete_schema.json` (git-ignored). In that copy each manifest site is `definitions/` with a `$ref` at the site. A merge with different bodies, or a closed-enum value that its base datatype rejects, fails the run. 2. **Codegen.** `datamodel-codegen --input ./build/codegen/oscal_complete_schema.json --output ./build/codegen/models.py --base-class oscal_bindings.v1._base.OscalBaseModel --collapse-root-models ... --formatters black isort` (see DEVELOPING "Names reach codegen through the hoisted schema" for the flag choices). Each `object`, `document`, and `enum` class takes its name from its definition key. `--base-class` makes all 185 object models subclass `OscalBaseModel`; the two `RootModel`s (`OscalDocument`, `MappingCollectionMappings`) don't. 3. **Post-process.** `python scripts/postprocess_models.py --models ./build/codegen/models.py --output src/oscal_bindings/v1/models.py`. Phase 1 collapses scalar `RootModel`s to `Annotated` aliases and strips `pattern=` from non-string fields (`AwareDatetime`/`AnyUrl`/numeric/`EmailStr`). Then it adds the 13 union `TypeAlias`es (`CatalogGroup`, plain unions, so `isinstance` works) and writes each field that uses one as `Annotated[, Field(union_mode='left_to_right')]` (the variant properties are exclusive, so this picks the same variant as smart mode, faster, #43), the `MappingCollectionMappings` root model, and the `OscalDocument` root model. It discriminates `OscalDocument.root` on the root key of the input: each member is `Tag`-ged with its class name, the root key of each member is the manifest `body-field`, and `root_key_discriminator` (`v1/_base.py`) selects the one member to validate (an input with no known root key is one `oscal_document_type` error). It names each scalar alias from its definition key without the `oscal-complete-oscal-:` prefix. Three guards compare the result with the manifest and the artifact schema and report every offender: each manifest name is defined once in the form of its kind, each other name is a scalar alias named for a scalar definition of the schema, no name is defined twice, and no manifest name or scalar alias is bound by a module-level import. The manifest must record the sha256 of the `--schema` bytes. The completed module goes to `--output` through a temp file + `os.replace`; on a failure nothing is written to `--output`, so a failed `generate` leaves the committed `models.py` untouched. `generate` passes no `--schema`: an omitted `--schema` resolves at call time to `artifact_paths(ensure_spec_artifact()).schema`, and an explicit one (a candidate schema, or a small test schema) must still match `schema.sha256` of the manifest. `--models` falls back to its own default, `--output` defaults to `--models`; a missing path is a non-zero exit. diff --git a/DEVELOPING.md b/DEVELOPING.md index d50c311..7e8fd0f 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -107,8 +107,8 @@ rule for a manifest type. `hatch run generate` runs three steps: 1. `scripts/hoist_names.py` writes `build/codegen/oscal_complete_schema.json` (git-ignored). 1. It fetches the spec artifact that `scripts/spec_artifact.py` pins (`0.8.0+oscal.1.2.3`, by sha256) through `scripts/pinned_archive.py`. - 2. `scripts/manifest.py` validates `names.json`: its JSON Schema from the same artifact (with `jsonschema-rs`), the - format version and OSCAL release, the sha256 of the artifact schema, unique names, and pointers that resolve. + 2. `scripts/manifest.py` validates `names.json`: the format version first (NAM-022), then its JSON Schema from the + same artifact (with `jsonschema-rs`), the OSCAL release, the sha256 of the artifact schema, unique names, and pointers that resolve. 3. It copies the complete OSCAL schema of the same artifact (`artifact_paths(...).schema`) and moves each manifest site into `definitions/`, with a `$ref` at each site. It reads the schema bytes one time: the manifest check hashes them, and the hoist parses the same bytes. A merge of sites with different bodies fails and names diff --git a/scripts/manifest.py b/scripts/manifest.py index 3a8307b..1fe8326 100644 --- a/scripts/manifest.py +++ b/scripts/manifest.py @@ -5,11 +5,13 @@ reject an invalid manifest before it uses one string of it. ``validate_manifest`` applies these checks in order and returns every failure: +0. ``names.json`` is a JSON object with a ``format`` member, and ``format`` + is a supported format version (NAM-021, NAM-022). A failure here stops the + validation: the other checks apply only to a format that this module reads. 1. ``names.json`` conforms to ``names.schema.json`` of the same artifact. A failure here stops the validation, because the later checks assume the shape that the JSON Schema defines. -2. ``format`` is a supported format version, and ``oscal-release`` is the - release of the schema that generation uses. +2. ``oscal-release`` is the release of the schema that generation uses. 3. ``schema.sha256`` is the sha256 of the schema bytes that generation reads. The caller gives these bytes, or ``load_validated_manifest`` reads them one time, so the check hashes the same bytes that the caller parses. @@ -281,6 +283,24 @@ def _cross_failures(manifest: dict[str, object], schema: object) -> list[str]: return failures +def _format_failure(manifest: object) -> str | None: + """Return the format version failure of ``manifest``, or ``None``. + + NAM-022: a consumer reads ``format`` before any JSON Schema check, and + states the unsupported value. NAM-021: a manifest that is not a JSON object, + or has no ``format`` member, is invalid. + """ + if not isinstance(manifest, dict): + return "[manifest] names.json is not a JSON object" + if "format" not in manifest: + return "[manifest] names.json has no format member" + fmt = manifest["format"] + if isinstance(fmt, bool) or fmt not in SUPPORTED_FORMATS: + supported = ", ".join(str(f) for f in SUPPORTED_FORMATS) + return f"[manifest] format {fmt!r} is not supported (supported: {supported})" + return None + + def validate_manifest( manifest: object, names_schema: dict[str, object], @@ -293,18 +313,13 @@ def validate_manifest( ``names.schema.json``. ``schema_bytes`` are the bytes of the complete OSCAL schema that codegen reads, and ``release`` is its OSCAL release. """ + format_failure = _format_failure(manifest) + if format_failure is not None: + return [format_failure] failures = _schema_failures(manifest, names_schema) - if failures: + if failures or not isinstance(manifest, dict): return failures - if not isinstance(manifest, dict): - return ["[manifest] names.json is not a JSON object"] - fmt = manifest.get("format") - if isinstance(fmt, bool) or fmt not in SUPPORTED_FORMATS: - supported = ", ".join(str(f) for f in SUPPORTED_FORMATS) - failures.append( - f"[manifest] format {fmt!r} is not supported (supported: {supported})" - ) oscal_release = manifest.get("oscal-release") if oscal_release != release: failures.append(f"[manifest] oscal-release {oscal_release!r} is not {release}") diff --git a/tests/test_manifest.py b/tests/test_manifest.py index a545152..53b734e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -184,23 +184,51 @@ def test_nam_003_04_item_and_member_references_must_name_an_entry( ) -def test_nam_003_05_unknown_format_version_is_rejected( +def test_nam_022_01_unsupported_format_version_is_stated( released: dict[str, Any], manifest: dict[str, Any] ) -> None: manifest["format"] = 2 - _assert_fails( - _validate(released, manifest), "[names.schema.json] names.json/format" - ) + assert _validate(released, manifest) == [ + "[manifest] format 2 is not supported (supported: 1)" + ] -def test_nam_003_05_unknown_format_version_is_rejected_without_json_schema( +def test_nam_022_02_format_version_is_the_first_check( released: dict[str, Any], manifest: dict[str, Any] ) -> None: manifest["format"] = 2 - _assert_fails( - _validate(released, manifest, names_schema={}), - "format 2 is not supported", - ) + manifest["unknown"] = True + manifest["oscal-release"] = "1.2.4" + assert _validate(released, manifest) == [ + "[manifest] format 2 is not supported (supported: 1)" + ] + + +@pytest.mark.parametrize("fmt", [True, "1", 1.5, None, [1]]) +def test_format_value_of_other_type_is_not_supported( + released: dict[str, Any], manifest: dict[str, Any], fmt: object +) -> None: + manifest["format"] = fmt + assert _validate(released, manifest) == [ + f"[manifest] format {fmt!r} is not supported (supported: 1)" + ] + + +def test_nam_021_01_manifest_without_format_is_invalid( + released: dict[str, Any], manifest: dict[str, Any] +) -> None: + del manifest["format"] + manifest["unknown"] = True + assert _validate(released, manifest) == [ + "[manifest] names.json has no format member" + ] + + +@pytest.mark.parametrize("value", [[], "names", 1, None]) +def test_nam_021_01_manifest_that_is_not_an_object_is_invalid( + released: dict[str, Any], value: object +) -> None: + assert _validate(released, value) == ["[manifest] names.json is not a JSON object"] def test_other_oscal_release_is_rejected(