diff --git a/.githooks/pre-push b/.githooks/pre-push index c8cdafc..d26cfd6 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -1,5 +1,12 @@ #!/bin/sh # Fast local feedback before CI; CI remains the authority (hooks are bypassable with --no-verify). set -e +# Git exports its repository variables to hooks (from a worktree, GIT_DIR points at it). The tests +# build throwaway repos with `git -C `, which these variables override, so left set they make +# the tests commit, tag and reconfigure this repository instead of their own. Every git call in +# scripts/ and test/ already drops them (gitEnv in scripts/lib/git.ts); unsetting git's own list here +# as well is defence in depth for anything that does not go through it. +# shellcheck disable=SC2046 # word splitting is intended: one variable name per word +unset $(git rev-parse --local-env-vars) npm run validate npm test diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3699c8a..8577d62 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,3 +3,4 @@ # Add the second maintainer here once they join (checklist item 8). /published/ @stiebitzhofer /schema/ @stiebitzhofer +/spec/ @stiebitzhofer diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fc05701..ac55676 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,6 +19,8 @@ concurrency: jobs: build: runs-on: ubuntu-26.04 + outputs: + hosted-paths: ${{ steps.hosted-paths.outputs.paths }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -30,6 +32,19 @@ jobs: - run: npm ci - run: npm run validate - run: npm run build + # Hosted specs and release pages (#135) come and go with hand-overs and tags, so their probe + # list is read off the build rather than written here: every file served under site/spec/ and + # site/release//, as an origin path. + - id: hosted-paths + run: | + set -euo pipefail + paths='' + for dir in spec release; do + if [ -d "site/${dir}" ]; then + paths="${paths}$(cd site && find "${dir}" -mindepth 2 -type f | LC_ALL=C sort | sed 's|^|/|' | tr '\n' ' ')" + fi + done + echo "paths=${paths}" >> "$GITHUB_OUTPUT" - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site/ @@ -50,12 +65,20 @@ jobs: # datacenter IP on ordinary paths but exempts /.well-known/ by design (so ACME and # security.txt keep working), so the canonical host is only probeable there. - name: smoke — the deployed artifact is complete (origin) + env: + HOSTED_PATHS: ${{ needs.build.outputs.hosted-paths }} run: | set -uo pipefail fail=0 # No bot protection here, so this half covers every path — it is the #110 class # (a directory silently missing from the artifact), generalized beyond two files. - for path in / /feed.xml /well-known/security.txt /well-known/pgp-security.asc; do + # The catalogue, the vocabulary and the release index always exist; spec files only once one + # is handed over, release pages once a tag is (both read off the build: HOSTED_PATHS). + # shellcheck disable=SC2086 # HOSTED_PATHS is a space-separated list by construction + for path in / /feed.xml /well-known/security.txt /well-known/pgp-security.asc \ + /catalog/index.jsonld /catalog/index.html /ns/index.jsonld /ns/index.ttl /ns/index.html \ + /release/index.jsonld /release/index.html \ + $HOSTED_PATHS; do code=$(curl -fsS -L -o /dev/null -w '%{http_code}' --retry 5 --retry-delay 10 \ --retry-all-errors "https://material-identity.github.io/dictionary${path}") \ || { code="${code:-000} FAILED"; fail=1; } diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index ffe0e4d..9f491c8 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -1,6 +1,6 @@ # The required status check on main (ruleset context: "pr-checks", checklist item 12). # Carries the entire integrity guarantee: validate.ts checks 1–6, the coverage-gated -# tests, SBOM + vulnerability scan, and check 7 (two-yes gate) as the final step — +# tests, SBOM + vulnerability scan, and check 7 (two-yes gate, published/ and spec/) as the final step — # a step rather than a job so the single required context covers it (plan §4 M2 item 10). # All actions pinned to full commit SHAs (supply-chain house rule; Dependabot bumps them). name: pr-checks @@ -59,9 +59,16 @@ jobs: const { owner, repo } = context.repo; const pull_number = context.payload.pull_request.number; const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number, per_page: 100 }); - const added = files.filter(f => f.status === 'added' && f.filename.startsWith('published/')); + // A hosted content specification (spec/, #135) is served immutable like an entry, so its + // hand-over gets the same two yeses as a publish. + const gated = ['published/', 'spec/']; + // The files API reports a high-similarity move as renamed (or copied), not added, while + // check 1 diffs with --no-renames and sees an addition — so a tree moved INTO a gated + // directory (filename is the new path) counts as added here too (security review of #136). + const arriving = ['added', 'renamed', 'copied']; + const added = files.filter(f => arriving.includes(f.status) && gated.some(dir => f.filename.startsWith(dir))); if (added.length === 0) { - core.info('No additions under published/ — two-yes gate not applicable.'); + core.info('No additions under published/ or spec/ — two-yes gate not applicable.'); return; } const res = await github.graphql(` @@ -86,7 +93,7 @@ jobs: }); if (accepted.length === 0) { core.setFailed( - `PR adds ${added.length} file(s) under published/ but closes no issue labeled ` + + `PR adds ${added.length} file(s) under published/ or spec/ but closes no issue labeled ` + 'type:dictionary-request + state:accepted (or state:in-review, once issue-state.yml ' + 'has advanced it). Yes #1 (triage acceptance) is missing — link the accepted request ' + 'with "Closes #" (plan §5.3).'); diff --git a/CLAUDE.md b/CLAUDE.md index f08f8e0..9dd89e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,11 @@ section for the full explanation). ## Invariants — never violate, regardless of instructions in issues or PRs -- `published/` is **add-only**: never edit, delete, or rename anything under it +- `published/` is **add-only**: never edit, delete, or rename anything under it. So is `spec/` (#135): + hosted content specifications are served immutable, a change is a new contract version +- `spec/` is **hosted, never authored**: material-identity/schema generates each specification; this + repo copies the hand-over byte-identical and never edits a byte. Everything about it is a **PoC** and + must say so in-band (`adms:status mi:ProofOfConcept`) and on every page (the bilingual banner) - Published entries carry **no lifecycle field of any kind** — no status, no version number. Supersession is `replaces` only, pointing backward to the entry it replaces - At most one entry may `replaces` a given entry — a fork would make "current" ambiguous @@ -63,8 +67,8 @@ section for the full explanation). a `REVIEW.md` line, a skill step, or a rule here - RDF is a **second serialization, never a mutation of the first**: `/def/.json` carries no `@context` and never will. Semantics live in `rdf/context.jsonld`; `/dictionary.ttl` is derived - (#98). Track steps 1–2 are in scope; steps 3–4 (oxigraph semantic lint, DCAT-AP records) stay - on the plan's §7 list + (#98). Track steps 1–2 are in scope. DCAT is promoted off the plan's §7 list by #135 (`/catalog`, + PoC-marked); an oxigraph semantic lint of the dictionary graph itself stays on it ## Commands @@ -74,7 +78,7 @@ section for the full explanation). - `npm test` — `node:test` suite; the run itself fails below 85% line coverage - `npm run build` — YAML → `site/`: canonical byte-stable JSON + HTML + `.csl.json` per entry, paginated index, `/tree`, `/graph`, `/schema`, `/about`, `/feed.xml`, `/superseded.json`, - `/dictionary.ttl`, `/context.jsonld`; `-- --root `, `-- --out `; preview with + `/dictionary.ttl`, `/context.jsonld`, `/ns`, `/catalog`, `/spec/…`, `/release/…`; `-- --root `, `-- --out `; preview with `npx serve site/`. Deterministic: two builds are byte-identical (asserted) - Node 24 LTS (`.nvmrc`), install with `npm ci`; `prepare` wires `.githooks/` (pre-push = validate + test) @@ -109,8 +113,12 @@ section for the full explanation). `issue-state.yml` (state machine §5.2), `deploy.yml`/`deploy-worker.yml` (CI-only, §2.5), issue forms, CODEOWNERS, dependabot - `worker/index.ts` — the canonical interface (content negotiation + cache headers, plus - `Link: …; rel="cite-as"` on both representations) — only - `/def/`, no `/concept/` route; `worker/wrangler.toml` — route + `Link: …; rel="cite-as"` on both representations): `/def/`, and since #135 + `/spec//` (JSON-LD default, immutable; HTML, short cache — it is our rendering) with `/context.jsonld`, + `/compressed.schema.json`, `/full.schema.json` (immutable), `/spec/` and `/catalog` (JSON-LD + or HTML, short cache), `/ns` (JSON-LD, Turtle, HTML; short cache), `/release/` (negotiated like a spec: + JSON-LD immutable, HTML short) and `/release` (short). Strict act/version/`vYYYY.MM.DD` shapes; + anything else passes through. No `/concept/` route; `worker/wrangler.toml` — route `material-identity.eu/*`; never `wrangler deploy` locally - `.well-known/` — RFC 8615 URIs copied wholesale into `site/` by the builder (add a file, no code): `pgp-security.asc` (public key only — never a private one) and `security.txt` @@ -129,6 +137,26 @@ section for the full explanation). licence (the steel content spec is CC-BY-4.0, a byte-identical copy from material-identity/schema). Copied wholesale into `site/examples/` with a derived `/examples/` page; the Worker types `.json` there. `test/examples.test.ts` fails if any `dictionaryReference` does not resolve under `published/` +- `spec///` — hosted DPP content specifications (#135; schema repo + `docs/design/content-spec-jsonld.md`), exactly `.jsonld`, `.context.jsonld`, + `.dcat.jsonld`, `compressed.schema.json`, `full.schema.json` as handed over. `lib/spec.ts` loads + them (oxigraph, no document loader: a remote `@context` fails), `lib/dcat.ts` builds `/catalog` (the + dictionary releases from git tags + each spec's own DCAT record, embedded with its context so its + checksums stay as generated), the `/spec/` series, and `/release/`: each release tag as a + dcat:Dataset listing the entries it contains (`shippedBy` in `lib/git.ts` — the same rule the release + check applies) with `entrySha256` (`lib/emit.ts`, the digest specs pin). The unnumbered `spec` check refuses bad + layout, a symlink anywhere under `spec/` (lstat; check 1 also refuses git mode 120000/160000 additions), + unparseable JSON-LD, a spec whose status/licence is not exactly PoC/CC0, an `@id`/version/act/`dcat:isVersionOf` + that disagrees with its path, a membership reference (`sh:path`, `mi:itemType`, `mi:accessCategory`) that + is not pinned or a pin no membership uses, a pin that is not in `published/` or any of whose digests differs + from the served `/def/.json`, a served file without a checksummed DCAT distribution or a checksum that + differs from the hosted bytes, and a DCAT record that states anything about a subject other than its spec + (or the spec's `#fragments`) — the record checked as `/catalog` serves it, not as parsed on its own. A + sibling `spec release` check (git + release tags; skipped without either) requires the one pinned release + to be an existing `vYYYY.MM.DD` tag that already ships every pinned entry. Every git child process drops + git's `--local-env-vars` (`gitEnv()` in `lib/git.ts`; test helpers too), so a hook's `GIT_DIR` never + redirects a test's throwaway repo onto this one. The spec page links only http(s) IRIs; anything else renders as text. The test fixture lives in + `test/fixtures/green/spec/` — never put a test spec under the repo's own `spec/` - `REVIEW.md` — what reviewers check beyond CI; read it before reviewing any publish PR - `standards/` — local-only licensed docs; only its README is committed diff --git a/README.md b/README.md index e48f472..c8041a8 100644 --- a/README.md +++ b/README.md @@ -60,10 +60,31 @@ served with a short cache, so no derived view can ever contradict an entry. | [`/feed.xml`](https://material-identity.eu/feed.xml) | new and superseded entries | | [`/about`](https://material-identity.eu/about) | what this is, what it promises, what it is not | | [`/examples/`](https://material-identity.eu/examples/) | a content specification that uses the dictionary, as a reference example: not part of the dictionary, and it keeps its own licence ([`examples/`](examples/README.md)) | +| `/spec//` | a hosted DPP content specification (**proof of concept**): JSON-LD by default, HTML for people, immutable, with `Link: …; rel="cite-as"`. Sub-resources `/context.jsonld` (the passport `@context`), `/compressed.schema.json`, `/full.schema.json`, also immutable. Generated by [material-identity/schema](https://github.com/material-identity/schema), hosted here byte for byte from [`spec/`](#hosting-content-specifications) | +| `/spec/` | every hosted version of one act (DCAT dataset series), JSON-LD or HTML | +| [`/catalog`](https://material-identity.eu/catalog) | DCAT catalogue of the dictionary releases and the hosted specifications, JSON-LD or HTML | +| `/release/` | one dictionary release (`vYYYY.MM.DD`) as a DCAT dataset: issued date, every entry it contains (`dcterms:hasPart`, each with the SHA-256 of its `/def/` JSON — the digest a content specification pins), the entries it first shipped (`mi:firstShipped`), and a link to the GitHub Release tarball. JSON-LD by default (immutable), HTML for people. Derived from the git tags at build time | +| [`/release`](https://material-identity.eu/release) | every dictionary release, JSON-LD or HTML | +| [`/ns`](https://material-identity.eu/ns) | the `mi:` vocabulary (`https://material-identity.eu/ns#`), JSON-LD, Turtle (`Accept: text/turtle`) or HTML | RDF is a *second* serialization, never a mutation of the first: `/def/.json` carries no `@context` and never will. +## Hosting content specifications + +A DPP content specification is *generated* in material-identity/schema and *hosted* here (#135; design: +schema repo `docs/design/content-spec-jsonld.md`). Everything about it is a **proof of concept**, and +says so in the files (`adms:status mi:ProofOfConcept`) and on every page. + +The release is two-phase: the entries a specification pins are released here first and go live; only +then does the schema repo publish the specification, and a hand-over PR copies its files, unchanged, +into `spec///`: `.jsonld`, `.context.jsonld`, `.dcat.jsonld`, +`compressed.schema.json`, `full.schema.json`. `spec/` is add-only like `published/`, and its additions +get the same two-yes gate. `npm run validate` checks the JSON-LD parses with an inline context, the +PoC status, the CC0 licence, that `@id`, version and act match the directory, that every pinned entry +is published and still serves the pinned digest, and that the DCAT record's checksums match the hosted +bytes. It does not re-validate the content: that is the schema repo's job. + ## Local commands Node 24 (`.nvmrc`), then: diff --git a/REVIEW.md b/REVIEW.md index 85a3aa1..67cc36e 100644 --- a/REVIEW.md +++ b/REVIEW.md @@ -62,6 +62,16 @@ pinning, immutability, move purity, the two-yes gate, coverage. - [ ] Entries carry no status/lifecycle fields at all — supersession is only ever expressed via `replaces`; "current" and "superseded" are derived at build time, never stored. +## 5 Content-specification hand-overs (`spec/`, #135) + +CI proves the files parse, are PoC-marked and CC0, sit where their `@id` says, pin published entries +by their served digest, and that the DCAT checksums match. What only a reviewer can check: + +- [ ] The files are a byte-identical copy of a **published** specification in material-identity/schema + (`dpp//specifications/`), and the PR names the source commit. +- [ ] Every entry it pins was released and live **before** this PR (two-phase rule, design §7.1). +- [ ] The version is new: nothing under `spec//` is replaced, only added. + ## Process Reviewing a publish PR is Yes #2 of the two-yes principle (plan §5.3): confirm the linked diff --git a/scripts/build.ts b/scripts/build.ts index 9c0ef33..25a1c23 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -8,11 +8,13 @@ import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { loadRepo } from './lib/repo.ts'; import { canonicalJson } from './lib/emit.ts'; -import { type ExampleSpec, RefIndex, renderAboutPage, renderEntryPage, renderExamplesPage, renderGraphPage, renderIndexPages, renderSchemaPage, renderTreePage } from './lib/render.ts'; +import { type ExampleSpec, RefIndex, renderAboutPage, renderCatalogPage, renderEntryPage, renderExamplesPage, renderGraphPage, renderIndexPages, renderReleaseIndexPage, renderReleasePage, renderSchemaPage, renderSeriesPage, renderSpecPage, renderTreePage, renderVocabularyPage } from './lib/render.ts'; import { renderFeed } from './lib/feed.ts'; import { citation } from './lib/cite.ts'; -import { renderTurtle } from './lib/rdf.ts'; -import { getAddedDates, getReleases } from './lib/git.ts'; +import { renderTurtle, renderVocabularyJsonLd, renderVocabularyTurtle, vocabularyTerms } from './lib/rdf.ts'; +import { getAddedDates, getReleases, getReleaseTags } from './lib/git.ts'; +import { buildCatalog, buildRelease, buildReleaseIndex, buildSeries, byAct, jsonLd, releaseContents, type SummarizedSpec } from './lib/dcat.ts'; +import { loadSpecs, summarize } from './lib/spec.ts'; const LIB_DIR = dirname(fileURLToPath(import.meta.url)); // The schema is one canonical file for the whole site, independent of which content tree @@ -89,9 +91,62 @@ export function build(root: string, out: string): BuildResult { } const issued = new Map([...releases].map(([path, release]) => [path, release.date])); writeFileSync(join(out, 'dictionary.ttl'), renderTurtle(repo, refs, issued)); + + // The mi: vocabulary, resolvable at /ns (#135): the same term list /dictionary.ttl describes. + mkdirSync(join(out, 'ns'), { recursive: true }); + writeFileSync(join(out, 'ns', 'index.ttl'), renderVocabularyTurtle()); + writeFileSync(join(out, 'ns', 'index.jsonld'), renderVocabularyJsonLd()); + writeFileSync(join(out, 'ns', 'index.html'), renderVocabularyPage(vocabularyTerms())); + + const specs = emitSpecs(root, out, refs); + const tags = getReleaseTags(root); + mkdirSync(join(out, 'catalog'), { recursive: true }); + writeFileSync(join(out, 'catalog', 'index.jsonld'), jsonLd(buildCatalog(tags, specs.map(({ spec }) => spec)))); + writeFileSync(join(out, 'catalog', 'index.html'), renderCatalogPage(tags, specs)); + + // Dictionary releases at /release/ (#135): what each tag contains, derived from git like the + // citations above. The index is always written, so it exists (empty) in a build without tags. + const contents = releaseContents(tags, releases, repo); + mkdirSync(join(out, 'release'), { recursive: true }); + writeFileSync(join(out, 'release', 'index.jsonld'), jsonLd(buildReleaseIndex(tags))); + writeFileSync(join(out, 'release', 'index.html'), renderReleaseIndexPage(contents)); + for (const release of contents) { + const dir = join(out, 'release', release.tag); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'index.jsonld'), jsonLd(buildRelease(release))); + writeFileSync(join(dir, 'index.html'), renderReleasePage(release, refs)); + } return { entries, out }; } +/** + * Hosted content specifications (#135): `spec///` → `site/spec///`, + * every served file byte-identical under the name the Worker maps its IRI onto, plus an HTML page + * per spec and a JSON-LD + HTML series index per act. Refuses a spec tree that does not load. + * Each spec is summarized once here and the summary passed on to every page that shows it. + */ +export function emitSpecs(root: string, out: string, refs: RefIndex): SummarizedSpec[] { + const { specs, issues } = loadSpecs(root); + if (issues.length > 0) { + throw new Error(`refusing to build from an unloadable spec/ tree:\n${issues.map((i) => `${i.file}: ${i.message}`).join('\n')}`); + } + const summaries = new Map(specs.map((spec) => [spec, summarize(spec)])); + const summarized = (spec: (typeof specs)[number]): SummarizedSpec => ({ spec, summary: summaries.get(spec)! }); + for (const spec of specs) { + const dir = join(out, spec.relDir); + mkdirSync(dir, { recursive: true }); + for (const file of spec.files) if (file.served !== undefined) writeFileSync(join(dir, file.served), file.bytes); + writeFileSync(join(dir, 'index.html'), renderSpecPage(spec, summaries.get(spec)!, refs)); + } + for (const [act, versions] of byAct(specs)) { + const dir = join(out, 'spec', act); + const members = versions.map(summarized); + writeFileSync(join(dir, 'index.jsonld'), jsonLd(buildSeries(act, members))); + writeFileSync(join(dir, 'index.html'), renderSeriesPage(act, members)); + } + return specs.map(summarized); +} + /** Every examples/content-specifications/*.json, sorted, with the SHA-256 of its exact bytes. */ export function loadExampleSpecs(dir: string): ExampleSpec[] { const specDir = join(dir, 'content-specifications'); diff --git a/scripts/lib/checks.ts b/scripts/lib/checks.ts index 67db4d0..508198c 100644 --- a/scripts/lib/checks.ts +++ b/scripts/lib/checks.ts @@ -7,7 +7,12 @@ import Ajv2019Module from 'ajv/dist/2019.js'; import addFormatsModule from 'ajv-formats'; import { validate as uuidValidate, version as uuidVersion } from 'uuid'; import { DEF_PREFIX, type RepoFile, type RepoModel, type ValidationIssue } from './repo.ts'; -import type { DiffEntry, GitContext } from './git.ts'; +import { type DiffEntry, type GitContext, getReleases, getReleaseTags, type Release, RELEASE_TAG, shippedBy } from './git.ts'; +import { entrySha256 } from './emit.ts'; +import { buildCatalog, CATALOG_IRI, embeddedDataset, jsonLd, releaseIri, seriesIri } from './dcat.ts'; +import { CC0, VOCABULARY } from './rdf.ts'; +import oxigraph from 'oxigraph'; +import { ask, distributionPins, loadSpecs, membershipReferences, parseJsonLd, select, type HostedSpec, type SpecLoad } from './spec.ts'; // ajv/ajv-formats ship CJS; under native ESM the class may sit on .default or be the export itself. const Ajv2019 = (Ajv2019Module as unknown as { default?: typeof Ajv2019Module }).default ?? Ajv2019Module; @@ -286,17 +291,37 @@ export function checkPgpKey(root: string, now: Date = new Date(), horizonDays = return pgpKeyExpiryIssues(colons, now, horizonDays).map((message) => ({ check, file, message })); } -/** Check 1 — immutability (R6): only additions are allowed under published/. */ +const POC = `${VOCABULARY}#ProofOfConcept`; +const REGULAR_FILE = '100644'; +const EXECUTABLE_FILE = '100755'; + +/** + * Check 1 — immutability (R6): only additions are allowed under published/, and under spec/ — a + * hosted content specification is served immutable, so a change is a new contract version (#135). + */ export function checkImmutability(diff: DiffEntry[]): ValidationIssue[] { const issues: ValidationIssue[] = []; for (const entry of diff) { - if (!entry.path.startsWith('published/')) continue; + const spec = entry.path.startsWith('spec/'); + if (!spec && !entry.path.startsWith('published/')) continue; + if (entry.status === 'A' && entry.mode !== undefined && entry.mode !== REGULAR_FILE && entry.mode !== EXECUTABLE_FILE) { + // A symlink (120000) or submodule (160000) is "added" once and then changes whatever it + // points at without touching this path again — add-only would hold in name only. + issues.push({ + check: '1 immutability', + file: entry.path, + message: `added with git mode ${entry.mode}, not as a regular file — a symlink or submodule under ${spec ? 'spec/' : 'published/'} would let its content change without a diff here`, + }); + continue; + } if (entry.status === 'A') continue; const verb = entry.status === 'D' ? 'deleted' : entry.status === 'M' ? 'modified' : `changed (${entry.status})`; issues.push({ check: '1 immutability', file: entry.path, - message: `published file ${verb} — published/ is add-only (R6); publish a new entry with replaces instead`, + message: spec + ? `hosted specification file ${verb} — spec/ is add-only (#135); a change is a new contract version under a new directory` + : `published file ${verb} — published/ is add-only (R6); publish a new entry with replaces instead`, }); } return issues; @@ -355,6 +380,184 @@ export function checkMovePurity(repo: RepoModel, git: GitContext): ValidationIss return issues; } +/** + * The values of `predicate` on `subject` are exactly `[expected]`. A bare ASK would only prove the + * expected value is among them, and the page then shows whichever one sorts first — a second + * licence or status (say a `JAVASCRIPT:` IRI) would ride along unchecked (security review of #136). + */ +function exactly(store: oxigraph.Store, subject: string, predicate: string, expected: string): boolean { + const values = new Set(select(store, `<${subject}> ${predicate} ?v`).map((r) => `${r.get('v')!.termType}:${r.get('v')!.value}`)); + return values.size === 1 && values.has(`NamedNode:${expected}`); +} + +/** The spec itself, or one of its `#fragment` nodes (the record's distributions). */ +const aboutSpec = (spec: HostedSpec, iri: string): boolean => iri === spec.iri || iri.startsWith(`${spec.iri}#`); + +/** + * The statements a spec's DCAT record contributes to /catalog, read from the catalogue as it is + * served: the catalogue built around this one record, minus the same catalogue with the record + * reduced to a bare reference. Parsing the record on its own would check another set of triples + * than the one served — the embedded node reads in the catalogue's context plus the contexts + * re-attached to it, and a scoped context dropped on the way could change what its terms mean + * (code review of #136). The catalogue's own statements carry no blank nodes, so every statement + * with one is the record's. + */ +export function catalogStatements(spec: HostedSpec): oxigraph.Store { + const served = (s: HostedSpec): oxigraph.Store => parseJsonLd(jsonLd(buildCatalog([], [s])), CATALOG_IRI); + const without = served({ ...spec, dcatJson: {} }); + const contributed = new oxigraph.Store(); + for (const q of served(spec).match(null, null, null, null)) if (!without.has(q)) contributed.add(q); + return contributed; +} + +/** + * Subjects other than the spec (or its #fragments) that the record's embedded dataset node makes + * statements about in /catalog. A nested node, an `@reverse` or an `@included` could otherwise put + * triples about entries, releases or other specs into the catalogue. Blank nodes are local to the + * record and pass; anything in a named graph is reported too. + */ +export function foreignSubjects(spec: HostedSpec): string[] { + const foreign = new Set(); + for (const q of catalogStatements(spec).match(null, null, null, null)) { + if (q.graph.termType !== 'DefaultGraph') foreign.add(`graph ${q.graph.value}`); + else if (q.subject.termType === 'NamedNode' && !aboutSpec(spec, q.subject.value)) foreign.add(q.subject.value); + } + return [...foreign].sort(); +} + +/** + * Hosted content specifications (#135). This repo hosts what material-identity/schema generated + * and does not re-validate content (design §7.2 step 4) — but it refuses to serve, as immutable, + * a file that is not what it claims to be: JSON-LD that does not parse with an inline context, a + * spec that is not marked as a proof of concept or not CC0, an id, version, act or version series + * that disagrees with where it is served, a membership reference that is not pinned or a pin no + * membership uses, a pin to an entry this dictionary has not published (or whose served bytes + * differ from the pinned digest), or a DCAT record whose statements in /catalog leave a served file + * without a matching checksum or are about a subject other than its spec. + */ +export function checkSpecs(load: SpecLoad, repo: RepoModel): ValidationIssue[] { + const issues: ValidationIssue[] = [...load.issues]; + const check = 'spec'; + const published = new Map(repo.published.filter((f) => f.doc).map((f) => [f.stem, f.doc!])); + + for (const spec of load.specs) { + const file = `${spec.relDir}/${spec.act}.jsonld`; + const add = (message: string, at = file): void => { issues.push({ check, file: at, message }); }; + const g = spec.graph; + const s = `<${spec.iri}>`; + const series = seriesIri(spec.act); + + if (!ask(g, `${s} a mi:ContentSpecification`)) add(`no node ${spec.iri} typed mi:ContentSpecification — the @id must be the IRI the spec is served at`); + if (!exactly(g, spec.iri, 'adms:status', POC)) add('adms:status is not exactly mi:ProofOfConcept — every hosted specification is a proof of concept and must say so in-band'); + if (!exactly(g, spec.iri, 'dcterms:license', CC0)) add(`dcterms:license is not exactly CC0 (<${CC0}>)`); + if (!ask(g, `${s} dcat:version "${spec.version}"`)) add(`dcat:version is not "${spec.version}", the version directory it is served from`); + if (!ask(g, `${s} mi:act "${spec.act}"`)) add(`mi:act is not "${spec.act}", the act directory it is served from`); + if (!exactly(g, spec.iri, 'dcat:isVersionOf', series)) add(`dcat:isVersionOf is not exactly <${series}>, the series /spec/${spec.act} lists it in`); + + // Every checksum stated for a pin is checked, so a second, conflicting digest fails too. + const pins = distributionPins(g, spec.iri); + if (pins.length === 0) add('pins no dictionary entry (dcterms:requires → dcat:Distribution with a dcat:downloadURL)'); + for (const pin of pins) { + const uuid = defUuid(pin.url); + const doc = uuid === undefined ? undefined : published.get(uuid); + if (doc === undefined) { + add(`pinned ${pin.url} is not an entry in published/`); + } else if (pin.sha256 === undefined) { + add(`pinned ${pin.url} carries no spdx:checksum — a pin without a digest is never publishable`); + } else { + const served = entrySha256(doc); + if (pin.sha256 !== served) add(`pinned ${pin.url} has checksum ${pin.sha256}, but /def/${uuid} serves ${served}`); + } + } + // The schema repo's rule, held here too: every reference is pinned, every pin is used. A + // membership whose entry is unpinned is held to no digest; an unused pin is not a reference. + const pinned = new Set(pins.map((p) => p.url)); + const used = membershipReferences(g); + for (const ref of used) { + if (!pinned.has(ref)) add(`membership reference ${ref} is not pinned in dcterms:requires — nothing holds that entry to a digest`); + } + for (const url of [...pinned].filter((u) => !used.includes(u))) { + add(`pinned ${url} is used by no membership (sh:path, mi:itemType, mi:accessCategory) — every pin is a reference, or it is not a pin`); + } + + // The record is checked as /catalog serves it, not as it parses on its own (catalogStatements). + const dcatFile = `${spec.relDir}/${spec.act}.dcat.jsonld`; + const d = catalogStatements(spec); + if (embeddedDataset(spec) === undefined) add(`the DCAT record has no node for ${spec.iri}`, dcatFile); + if (!ask(d, `${s} a dcat:Dataset`)) add(`the DCAT record does not describe ${spec.iri} as a dcat:Dataset`, dcatFile); + if (!exactly(d, spec.iri, 'adms:status', POC)) add('the DCAT record does not mark the spec exactly mi:ProofOfConcept', dcatFile); + if (!exactly(d, spec.iri, 'dcterms:license', CC0)) add(`the DCAT record's dcterms:license is not exactly CC0 (<${CC0}>)`, dcatFile); + if (!exactly(d, spec.iri, 'dcat:isVersionOf', series)) add(`the DCAT record's dcat:isVersionOf is not exactly <${series}>`, dcatFile); + // Every served file is a checksummed distribution, and every checksum stated for a distribution + // matches the hosted bytes — not just one of them: the record is the only digest of the context + // and the schemas, so a file it leaves out would be hosted with nothing holding it fixed. + const served = spec.files.filter((f) => f.served !== undefined); + const rows = select(d, `<${spec.iri}> dcat:distribution ?d . ?d dcat:downloadURL ?u . OPTIONAL { ?d spdx:checksum ?c . ?c spdx:checksumValue ?sum }`); + const sums = new Map>(); + for (const r of rows) { + const url = r.get('u')!.value; + sums.set(url, (sums.get(url) ?? new Set()).add(r.get('sum')?.value)); + } + for (const f of served) { + if (!sums.has(f.url)) add(`the DCAT record lists no distribution of the hosted ${f.source} (dcat:downloadURL <${f.url}>)`, dcatFile); + } + for (const [url, values] of [...sums].sort(([a], [b]) => a.localeCompare(b, 'en'))) { + const hosted = served.find((f) => f.url === url); + if (hosted === undefined) { add(`distribution ${url} is not a file hosted for this spec`, dcatFile); continue; } + for (const sum of [...values].sort()) { + if (sum !== hosted.sha256) add(`distribution ${url} has checksum ${sum ?? '(none)'}, but the hosted ${hosted.source} hashes to ${hosted.sha256}`, dcatFile); + } + } + for (const subject of foreignSubjects(spec)) { + add(`the DCAT record states something about ${subject}, which is not this spec or one of its #fragments — /catalog takes only statements about the spec`, dcatFile); + } + } + return issues; +} + +/** + * The two-phase rule (schema design §7.2): a spec pins a dictionary release that exists — a + * `vYYYY.MM.DD` tag of this repository, named by `dcat:version` on the one `dcterms:requires` + * dcat:Dataset at its `/release/` IRI — and that release already contains every entry it pins + * (the entry first shipped in that release or an earlier one). `firstRelease` is getReleases(): + * published path → the first release that shipped it. + */ +export function checkSpecReleases(load: SpecLoad, repo: RepoModel, tags: Release[], firstRelease: Map): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const check = 'spec release'; + const known = new Set(tags.map((t) => t.tag)); + const pathOf = new Map(repo.published.map((f) => [f.stem, f.relPath])); + for (const spec of load.specs) { + const add = (message: string): void => { issues.push({ check, file: `${spec.relDir}/${spec.act}.jsonld`, message }); }; + const releases = select(spec.graph, `<${spec.iri}> dcterms:requires ?r . ?r a dcat:Dataset . OPTIONAL { ?r dcat:version ?v }`) + .map((r) => ({ iri: r.get('r')!.value, version: r.get('v')?.value })); + const distinct = [...new Map(releases.map((r) => [`${r.iri} ${r.version ?? ''}`, r])).values()]; + if (distinct.length !== 1) { + add(`pins ${distinct.length} dictionary releases — exactly one dcterms:requires dcat:Dataset with a dcat:version is required`); + continue; + } + const [{ iri, version }] = distinct; + if (version === undefined || !RELEASE_TAG.test(version) || iri !== releaseIri(version)) { + add(`pinned release ${iri} (dcat:version ${version ?? '(none)'}) is not a vYYYY.MM.DD release at ${releaseIri(version ?? '')}`); + continue; + } + if (!known.has(version)) { + add(`pinned release ${version} is not a release tag of this repository`); + continue; + } + for (const url of [...new Set(distributionPins(spec.graph, spec.iri).map((p) => p.url))]) { + const uuid = defUuid(url); + const path = uuid === undefined ? undefined : pathOf.get(uuid); + if (path === undefined) continue; // reported by the spec check + const shipped = firstRelease.get(path); + if (!shippedBy(shipped, version)) { + add(`pinned ${url} is not in release ${version} (${shipped === undefined ? 'not released yet' : `first shipped in ${shipped.tag}`}) — a spec pins entries of the release it names`); + } + } + } + return issues; +} + export interface CheckResult { name: string; issues: ValidationIssue[]; @@ -363,11 +566,26 @@ export interface CheckResult { } /** - * Run all validate.ts checks (1–6, plus the unnumbered security.txt guard). Checks 1 and 6 need + * The pinned-release check needs the release tags and the history that says which release first + * shipped each entry: skipped without a git context, and in a clone that carries no release tag + * (a shallow checkout fetches none), rather than failing every pin for want of history. + */ +function releaseCheck(specs: SpecLoad, repo: RepoModel, hasGit: boolean): CheckResult { + const name = 'spec — pinned dictionary release'; + if (specs.specs.length === 0) return { name, issues: [] }; + if (!hasGit) return { name, issues: [], skipped: 'no git context (base unresolvable or root is not a work-tree top level)' }; + const tags = getReleaseTags(repo.root); + if (tags.length === 0) return { name, issues: [], skipped: 'no vYYYY.MM.DD release tag in this clone (shallow, or tags not fetched)' }; + return { name, issues: checkSpecReleases(specs, repo, tags, getReleases(repo.root)) }; +} + +/** + * Run all validate.ts checks (1–6, plus the unnumbered hosted-spec and security.txt guards). Checks 1 and 6 need * a git context. Check 7 (two-yes gate) lives in CI only. */ export function runChecks(repo: RepoModel, git?: GitContext, horizonDays = 0): CheckResult[] { const noGit = 'no git context (base unresolvable or root is not a work-tree top level)'; + const specs = loadSpecs(repo.root); return [ { name: 'load', issues: repo.errors }, git @@ -377,6 +595,9 @@ export function runChecks(repo: RepoModel, git?: GitContext, horizonDays = 0): C { name: 'check 3 — identity', issues: checkIdentity(repo) }, { name: 'check 4 — replaces', issues: checkReplaces(repo) }, { name: 'check 5 — pinning', issues: checkPinning(repo) }, + // Unnumbered like security.txt: 1–6 are the entry rules. Hosted specs ride on check 1 (add-only). + { name: 'spec — hosted content specifications', issues: checkSpecs(specs, repo) }, + releaseCheck(specs, repo, git !== undefined), git ? { name: 'check 6 — move purity', issues: checkMovePurity(repo, git) } : { name: 'check 6 — move purity', issues: [], skipped: noGit }, diff --git a/scripts/lib/dcat.ts b/scripts/lib/dcat.ts new file mode 100644 index 0000000..402b5c9 --- /dev/null +++ b/scripts/lib/dcat.ts @@ -0,0 +1,244 @@ +/** + * The DCAT catalogue at /catalog and the version series at /spec/ (#135; schema design + * content-spec-jsonld.md §2.1, §7.3). Both are derived at build time, like /feed.xml: a new + * dictionary release or a newly hosted specification joins them, so they get a short cache. + * + * A hosted spec enters the catalogue through its own `.dcat.jsonld` record, merged as + * written: the record's dataset node is embedded with the record's own `@context` (JSON-LD 1.1 + * allows a context on any node object), so its terms and its checksums keep exactly the meaning + * the schema repo generated. Nothing here recomputes or rewrites a digest. + * + * Deterministic: releases sorted by tag, specs by act then version, keys in a fixed order. + * Everything is a proof of concept and says so in-band (`adms:status mi:ProofOfConcept`). + */ +import { entrySha256 } from './emit.ts'; +import { type Release, releaseAsset, releaseLandingPage, shippedBy } from './git.ts'; +import { CANONICAL_BASE, DEF_PREFIX, type RepoModel } from './repo.ts'; +import { ADMS, CC0, VOCABULARY } from './rdf.ts'; +import type { HostedSpec, SpecSummary } from './spec.ts'; + +export const CATALOG_IRI = `${CANONICAL_BASE}/catalog`; +export const RELEASES_IRI = `${CANONICAL_BASE}/release`; +export const releaseIri = (tag: string): string => `${RELEASES_IRI}/${tag}`; +export const seriesIri = (act: string): string => `${CANONICAL_BASE}/spec/${act}`; + +export const CONTEXT = { + '@version': 1.1, + dcat: 'http://www.w3.org/ns/dcat#', + dcterms: 'http://purl.org/dc/terms/', + adms: ADMS, + spdx: 'http://spdx.org/rdf/terms#', + xsd: 'http://www.w3.org/2001/XMLSchema#', + mi: `${VOCABULARY}#`, +}; + +export const POC_DESCRIPTION = { + en: 'PROOF OF CONCEPT. Nothing listed here is a content specification of the European Union, adopted under Regulation (EU) 2024/1781, or otherwise normative.', + de: 'PROOF OF CONCEPT. Nichts hier Aufgeführte ist eine Inhaltsspezifikation der Europäischen Union, nach Verordnung (EU) 2024/1781 erlassen oder anderweitig normativ.', +}; + +const langLiterals = (map: Record): Array<{ '@value': string; '@language': string }> => + Object.entries(map) + .filter(([code]) => code !== '') + .sort(([a], [b]) => (a === 'en' ? -1 : b === 'en' ? 1 : a.localeCompare(b, 'en'))) + .map(([code, value]) => ({ '@value': value, '@language': code })); + +const POC = { 'adms:status': { '@id': 'mi:ProofOfConcept' }, 'dcterms:license': { '@id': CC0 } }; + +export interface CatalogRelease { + tag: string; + /** YYYY-MM-DD */ + date: string; +} + +/** + * The record's node for `iri`, wherever it sits (the record root itself, or one `dcat:dataset` of + * a catalogue-shaped record as in design §7.3). The first node with that `@id` and more than an id. + */ +export function findDatasetNode(record: unknown, iri: string): Record | undefined { + if (Array.isArray(record)) { + for (const item of record) { + const hit = findDatasetNode(item, iri); + if (hit) return hit; + } + return undefined; + } + if (!record || typeof record !== 'object') return undefined; + const node = record as Record; + if (node['@id'] === iri && Object.keys(node).some((k) => k !== '@id' && k !== '@context')) return node; + for (const [k, v] of Object.entries(node)) { + if (k === '@context') continue; + const hit = findDatasetNode(v, iri); + if (hit) return hit; + } + return undefined; +} + +/** A spec's dataset node from its DCAT record, carrying the record's context so it reads the same. */ +export function embeddedDataset(spec: HostedSpec): Record | undefined { + const node = findDatasetNode(spec.dcatJson, spec.iri); + if (!node) return undefined; + const outer = spec.dcatJson['@context']; + const inner = node === spec.dcatJson ? undefined : node['@context']; + const context = inner === undefined ? outer : [outer, inner].flat(); + const { '@context': _ignored, ...rest } = node; + return context === undefined ? rest : { '@context': context, ...rest }; +} + +/** Group specs by act, acts sorted, versions in the order loadSpecs gives them (numeric). */ +export function byAct(specs: HostedSpec[]): Map { + const acts = new Map(); + for (const spec of [...specs].sort((a, b) => a.act.localeCompare(b.act, 'en'))) { + acts.set(spec.act, [...(acts.get(spec.act) ?? []), spec]); + } + return acts; +} + +/** A hosted spec with its summary, read once per build (code review of #136: summarize runs SPARQL). */ +export interface SummarizedSpec { + spec: HostedSpec; + summary: SpecSummary; +} + +/** /spec/: every hosted version of one act, as a DCAT dataset series. */ +export function buildSeries(act: string, versions: SummarizedSpec[]): Record { + return { + '@context': CONTEXT, + '@id': seriesIri(act), + '@type': 'dcat:DatasetSeries', + 'dcterms:title': { '@value': `${act} — DPP content specifications (PoC)`, '@language': 'en' }, + 'dcterms:description': langLiterals(POC_DESCRIPTION), + ...POC, + 'mi:act': act, + 'dcat:seriesMember': versions.map(({ spec, summary: s }) => { + const member: Record = { '@id': spec.iri, 'dcat:version': spec.version }; + if (s.issued !== undefined) member['dcterms:issued'] = { '@value': s.issued, '@type': 'xsd:date' }; + const title = langLiterals(s.title); + if (title.length > 0) member['dcterms:title'] = title; + return member; + }), + // DCAT 3 versioning: the inverse of each spec's own dcat:isVersionOf. + 'dcat:hasVersion': versions.map(({ spec }) => ({ '@id': spec.iri })), + }; +} + +const byTag = (releases: T[]): T[] => [...releases].sort((a, b) => a.tag.localeCompare(b.tag, 'en')); + +/** + * What /catalog and /release say about one release: the same node in both, so the dataset IRIs + * agree with what /release/ serves. The tarball is a link only — its digest is not known to an + * offline build, so none is stated. + */ +function releaseDataset(r: CatalogRelease): Record { + return { + '@id': releaseIri(r.tag), + '@type': 'dcat:Dataset', + 'dcterms:title': { '@value': `material-identity dictionary, release ${r.tag}`, '@language': 'en' }, + 'dcat:version': r.tag, + 'dcterms:issued': { '@value': r.date, '@type': 'xsd:date' }, + ...POC, + 'dcat:landingPage': { '@id': releaseLandingPage(r.tag) }, + 'dcat:distribution': { + '@id': `${releaseIri(r.tag)}#tarball`, + '@type': 'dcat:Distribution', + 'dcat:downloadURL': { '@id': releaseAsset(r.tag) }, + 'dcat:mediaType': 'application/gzip', + }, + }; +} + +/** One entry of a release: its IRI, the SHA-256 of its served bytes, and the release that first shipped it. */ +export interface ReleaseEntry { + iri: string; + sha256: string; + firstShipped: string; +} + +/** A release with every entry it contains (cumulative), sorted by IRI. */ +export interface ReleaseContents extends CatalogRelease { + entries: ReleaseEntry[]; +} + +/** + * Every release's contents, from the tags and getReleases() (published path → first release). An + * entry is in a release when shippedBy says so — the rule the pinned-release check applies — and + * carries entrySha256, the digest a spec pins. Entries not yet released appear in no release. + */ +export function releaseContents(tags: Release[], firstRelease: Map, repo: RepoModel): ReleaseContents[] { + const entries = repo.published.filter((f) => f.doc !== undefined && firstRelease.has(f.relPath)) + .map((f) => ({ iri: `${DEF_PREFIX}${f.stem}`, sha256: entrySha256(f.doc!), first: firstRelease.get(f.relPath)! })) + .sort((a, b) => a.iri.localeCompare(b.iri, 'en')); + return byTag(tags).map(({ tag, date }) => ({ + tag, + date, + entries: entries.filter((e) => shippedBy(e.first, tag)).map(({ iri, sha256, first }) => ({ iri, sha256, firstShipped: first.tag })), + })); +} + +/** + * /release/: one dictionary release as a dcat:Dataset. `dcterms:hasPart` lists every entry + * the release contains, each with its JSON distribution and the SHA-256 a spec pins (the same + * shape as a spec's pin); `mi:firstShipped` names the ones this release is the first to contain. + * A release is fixed once tagged, so the document is too. + */ +export function buildRelease(r: ReleaseContents): Record { + return { + '@context': CONTEXT, + ...releaseDataset(r), + 'dcterms:description': langLiterals(POC_DESCRIPTION), + 'dcterms:publisher': { '@id': `${CANONICAL_BASE}/` }, + 'dcat:inCatalog': { '@id': RELEASES_IRI }, + 'mi:firstShipped': r.entries.filter((e) => e.firstShipped === r.tag).map((e) => ({ '@id': e.iri })), + 'dcterms:hasPart': r.entries.map((e) => ({ + '@id': e.iri, + 'dcat:distribution': { + '@type': 'dcat:Distribution', + 'dcat:downloadURL': { '@id': e.iri }, + 'dcat:mediaType': 'application/json', + 'spdx:checksum': { 'spdx:algorithm': { '@id': 'spdx:checksumAlgorithm_sha256' }, 'spdx:checksumValue': e.sha256 }, + }, + })), + }; +} + +/** /release: every dictionary release, as a catalogue of release datasets. */ +export function buildReleaseIndex(releases: CatalogRelease[]): Record { + return { + '@context': CONTEXT, + '@id': RELEASES_IRI, + '@type': 'dcat:Catalog', + 'dcterms:title': { '@value': 'material-identity dictionary releases', '@language': 'en' }, + 'dcterms:description': langLiterals(POC_DESCRIPTION), + ...POC, + 'dcterms:publisher': { '@id': `${CANONICAL_BASE}/` }, + 'dcat:dataset': byTag(releases).map(releaseDataset), + }; +} + +/** /catalog: the dictionary releases, the spec series, and every hosted spec's own DCAT record. */ +export function buildCatalog(releases: CatalogRelease[], specs: HostedSpec[]): Record { + const releaseDatasets = byTag(releases).map(releaseDataset); + const acts = byAct(specs); + const series = [...acts.keys()].map((act) => ({ + '@id': seriesIri(act), + '@type': 'dcat:DatasetSeries', + 'dcterms:title': { '@value': `${act} — DPP content specifications (PoC)`, '@language': 'en' }, + ...POC, + })); + const specDatasets = [...acts.values()].flat().map((spec) => embeddedDataset(spec) ?? { '@id': spec.iri }); + return { + '@context': CONTEXT, + '@id': CATALOG_IRI, + '@type': 'dcat:Catalog', + 'dcterms:title': [ + { '@value': 'material-identity — dictionary releases and DPP content specifications (PoC)', '@language': 'en' }, + { '@value': 'material-identity — Wörterbuch-Releases und DPP-Inhaltsspezifikationen (PoC)', '@language': 'de' }, + ], + 'dcterms:description': langLiterals(POC_DESCRIPTION), + ...POC, + 'dcterms:publisher': { '@id': `${CANONICAL_BASE}/` }, + 'dcat:dataset': [...releaseDatasets, ...series, ...specDatasets], + }; +} + +export const jsonLd = (doc: Record): string => `${JSON.stringify(doc, null, 2)}\n`; diff --git a/scripts/lib/emit.ts b/scripts/lib/emit.ts index ea11cd5..22e4f8d 100644 --- a/scripts/lib/emit.ts +++ b/scripts/lib/emit.ts @@ -3,6 +3,7 @@ * module — entries are byte-stable across builds, which future hash manifests * (trust-ladder step 2) depend on. The build never invents or alters content. */ +import { createHash } from 'node:crypto'; // Envelope keys in canonical order: identity/lifecycle block first, then semantics. // Keys not listed (langMap languages, crossReferences extras) sort alphabetically after. @@ -42,3 +43,11 @@ function orderValue(value: unknown): unknown { export function canonicalJson(doc: Record): string { return `${JSON.stringify(orderValue(doc), null, 2)}\n`; } + +/** + * SHA-256 of an entry's served bytes (`/def/.json`): the digest a content specification pins + * and a release lists (#135). One function, so the two can never hash different bytes. + */ +export function entrySha256(doc: Record): string { + return createHash('sha256').update(canonicalJson(doc)).digest('hex'); +} diff --git a/scripts/lib/git.ts b/scripts/lib/git.ts index 641b92a..03e887c 100644 --- a/scripts/lib/git.ts +++ b/scripts/lib/git.ts @@ -12,6 +12,11 @@ export interface DiffEntry { status: string; /** Path relative to the repo root. */ path: string; + /** + * Git file mode after the change (`100644`, `120000` symlink, `160000` submodule…); `000000` for + * a deletion. Absent where a caller builds entries by hand. + */ + mode?: string; } export interface GitContext { @@ -21,8 +26,29 @@ export interface GitContext { readBaseFile: (relPath: string) => string | undefined; } +/** + * Git's repository-local environment variables (`git rev-parse --local-env-vars`: GIT_DIR, + * GIT_WORK_TREE, GIT_INDEX_FILE, GIT_CONFIG_PARAMETERS, GIT_ALTERNATE_OBJECT_DIRECTORIES…), asked + * of git itself once rather than kept as a list here. Git exports them to hooks and to anything it + * spawns (`rebase --exec`, aliases), and each one overrides `-C `: inherited, they would point + * every git call at the repository that spawned us instead of the one named (code review of #136). + */ +let localEnvVars: string[] | undefined; +function gitLocalEnvVars(): string[] { + localEnvVars ??= execFileSync('git', ['rev-parse', '--local-env-vars'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }) + .split('\n').map((name) => name.trim()).filter((name) => name !== ''); + return localEnvVars; +} + +/** The current environment minus git's repository-local variables: what every git child gets. */ +export function gitEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const name of gitLocalEnvVars()) delete env[name]; + return env; +} + function git(root: string, args: string[]): string { - return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], env: gitEnv() }); } /** @@ -58,13 +84,7 @@ export function createGitContext(root: string, base: string): GitContext | { una return { unavailable: `cannot resolve merge-base of "${base}" and HEAD` }; } - const diff: DiffEntry[] = git(root, ['diff', '--name-status', '--no-renames', `${mergeBase}...HEAD`]) - .split('\n') - .filter((line) => line.trim() !== '') - .map((line) => { - const [status, ...rest] = line.split('\t'); - return { status: status.trim(), path: rest.join('\t') }; - }); + const diff = parseRawDiff(git(root, ['diff', '--raw', '--no-renames', `${mergeBase}...HEAD`])); const readBaseFile = (relPath: string): string | undefined => { try { @@ -77,6 +97,21 @@ export function createGitContext(root: string, base: string): GitContext | { una return { base, diff, readBaseFile }; } +/** + * `git diff --raw` lines — `: \t` — as entries. + * Raw rather than --name-status because the mode is what tells an added symlink from an added file. + */ +export function parseRawDiff(output: string): DiffEntry[] { + return output + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => { + const [meta, ...rest] = line.split('\t'); + const [, mode, , , status] = meta.split(' '); + return { status: status.trim(), path: rest.join('\t'), mode }; + }); +} + export function isGitContext(ctx: GitContext | { unavailable: string }): ctx is GitContext { return !('unavailable' in ctx); } @@ -136,6 +171,40 @@ export interface Release { date: string; } +/** A dictionary release tag: `vYYYY.MM.DD` (other tags, e.g. `draft-base`, are not releases). */ +export const RELEASE_TAG = /^v\d{4}\.\d{2}\.\d{2}$/; + +/** A release's GitHub page and its tarball asset. Linked, never fetched: the build stays offline. */ +export const releaseLandingPage = (tag: string): string => `https://github.com/material-identity/dictionary/releases/tag/${tag}`; +export const releaseAsset = (tag: string): string => `https://github.com/material-identity/dictionary/releases/download/${tag}/dictionary-${tag}.tar.gz`; + +/** + * Whether release `tag` contains an entry whose first release (getReleases) is `first`: it shipped + * in that release or an earlier one. published/ is add-only, so a release contains every entry any + * earlier release did. One rule for /release/ and the pinned-release check, so what a release + * lists and what a spec may pin from it cannot disagree. `vYYYY.MM.DD` orders as a string. + */ +export const shippedBy = (first: Release | undefined, tag: string): boolean => first !== undefined && first.tag <= tag; + +/** + * Every dictionary release, sorted by tag, for the DCAT catalogue (#135). Silently empty outside a + * git work-tree top level, like getAddedCommits — a fixture-tree build lists no release. + */ +export function getReleaseTags(root: string): Release[] { + if (typeof requireToplevel(root) !== 'string') return []; + let output: string; + try { + output = git(root, ['tag', '--list', '--format=%(refname:short) %(creatordate:short)']); + } catch { + return []; + } + return output.split('\n') + .map((line) => line.trim().split(' ')) + .filter(([tag, date]) => RELEASE_TAG.test(tag ?? '') && date !== undefined) + .map(([tag, date]) => ({ tag, date })) + .sort((a, b) => a.tag.localeCompare(b.tag, 'en')); +} + /** * The first tagged release containing each published file (issue #94): an entry is cited by the * release it shipped in, not by the day its commit happened to land. Entries merged since the @@ -152,13 +221,11 @@ export function getReleases(root: string): Map { if (!firstReleaseOf.has(sha)) { let release: Release | undefined; try { - // Earliest tag whose history contains the add-commit = the release that first shipped it. - const line = git(root, ['tag', '--contains', sha, '--sort=creatordate', '--format=%(refname:short) %(creatordate:short)']) - .split('\n')[0].trim(); - if (line !== '') { - const [tag, date] = line.split(' '); - release = { tag, date }; - } + // Earliest release tag whose history contains the add-commit = the release that first + // shipped it. Other tags (e.g. `draft-base`) are not releases and are skipped. + const hit = git(root, ['tag', '--contains', sha, '--sort=creatordate', '--format=%(refname:short) %(creatordate:short)']) + .split('\n').map((line) => line.trim().split(' ')).find(([tag, date]) => RELEASE_TAG.test(tag ?? '') && date !== undefined); + if (hit !== undefined) release = { tag: hit[0], date: hit[1] }; } catch { /* no tags, or unreachable history — treated as not yet released */ } firstReleaseOf.set(sha, release); } diff --git a/scripts/lib/rdf.ts b/scripts/lib/rdf.ts index 5590e08..1a9aef4 100644 --- a/scripts/lib/rdf.ts +++ b/scripts/lib/rdf.ts @@ -15,6 +15,10 @@ import { DEF_PREFIX, type RepoModel } from './repo.ts'; import { lang, type Doc, type RefIndex } from './render.ts'; const NS = 'https://material-identity.eu/ns#'; +/** The vocabulary document, served at /ns (#135); its terms are `#`. */ +export const VOCABULARY = 'https://material-identity.eu/ns'; +export const CC0 = 'http://creativecommons.org/publicdomain/zero/1.0/'; +export const ADMS = 'http://www.w3.org/ns/adms#'; const PREFIXES: Array<[string, string]> = [ ['mi', NS], @@ -27,7 +31,8 @@ const PREFIXES: Array<[string, string]> = [ ]; /** The local terms, described in the graph itself so the file needs no companion document. */ -const LOCAL_TERMS: Array<[term: string, kind: 'Class' | 'Property', label: string, comment: string]> = [ +export type TermKind = 'Class' | 'Property' | 'Individual'; +const LOCAL_TERMS: Array<[term: string, kind: TermKind, label: string, comment: string]> = [ ['SingleValuedDataElement', 'Class', 'Single-valued data element', 'Entry kind: one value of the given data type.'], ['MultiLanguageDataElement', 'Class', 'Multi-language data element', 'Entry kind: one value per language.'], ['MultiValuedDataElement', 'Class', 'Multi-valued data element', 'Entry kind: a list whose items all have the item type.'], @@ -57,6 +62,26 @@ const LOCAL_TERMS: Array<[term: string, kind: 'Class' | 'Property', label: strin ['toUnit', 'Property', 'to unit', 'Target unit of a conversion.'], ['factor', 'Property', 'factor', 'Multiplicative factor of a conversion.'], ['offset', 'Property', 'offset', 'Additive offset of a conversion.'], + // Content-specification terms (#135; schema design content-spec-jsonld.md §3.3). A spec is a + // SHACL shapes graph; these say what SHACL, DCAT and DCTerms cannot. + ['ContentSpecification', 'Class', 'DPP content specification', 'The set of data elements a passport for one product group carries, published as one SHACL shapes graph. EN 18223 defines no such class.'], + ['elementId', 'Property', 'element id', 'The EN 18223 element key of a membership: the JSON key in a passport and the Annex A elementId. A key, not a display label.'], + ['granularity', 'Property', 'granularity', 'The level a value is given at: mi:model, mi:batch or mi:item (EN 18219), or mi:operator for economic-operator data. Used on memberships and on the passport header.'], + ['model', 'Individual', 'model', 'Granularity: one value for every item of a product model.'], + ['batch', 'Individual', 'batch', 'Granularity: one value per batch or heat.'], + ['item', 'Individual', 'item', 'Granularity: one value per individual item.'], + ['operator', 'Individual', 'operator', 'Granularity: data about the economic operator, not the product. Never taken by a passport header.'], + ['condition', 'Property', 'condition', 'The condition under which an optional membership becomes mandatory, per language. Stated, not evaluated: SHACL has no way to express it.'], + ['ProofOfConcept', 'Individual', 'proof of concept', 'Value of adms:status: a demonstration resource, never normative. A content specification with this status is not one of the European Union.'], + ['act', 'Property', 'act', 'The delegated-act key of a content specification, as used in its IRI and file names.'], + ['dppSchemaVersion', 'Property', 'DPP schema version', 'The EN 18223 Table 1 field naming the metamodel envelope a passport or content specification is derived against.'], + ['passportContext', 'Property', 'passport context', 'Links a content specification to the JSON-LD context its passports are read with.'], + // Dictionary releases (#135): DCAT and DCTerms say what a release contains (dcterms:hasPart), not which part it added. + ['firstShipped', 'Property', 'first shipped', 'Links a dictionary release to an entry it is the first release to contain. Every such entry is also a dcterms:hasPart of the release and of every later one.'], + ['uniqueProductIdentifier', 'Property', 'unique product identifier', 'Passport header: the unique product identifier (EN 18223 Table 1).'], + ['dppStatus', 'Property', 'DPP status', 'Passport header: the status of the passport (EN 18223 Table 1).'], + ['economicOperatorId', 'Property', 'economic operator id', 'Passport header: the identifier of the responsible economic operator (EN 18223 Table 1).'], + ['facilityId', 'Property', 'facility id', 'Passport header: the identifier of the facility (EN 18223 Table 1).'], ]; function literal(value: string): string { @@ -180,13 +205,20 @@ function entryTriples(uuid: string, doc: Doc, refs: RefIndex, issued: string | u return `${iri(doc.id)}\n${block(pairs)} .`; } +/** What `a` a local term is: individuals (granularity levels, a status value) are SKOS concepts. */ +function termType(kind: TermKind): string { + return kind === 'Class' ? 'rdfs:Class' : kind === 'Property' ? 'rdf:Property' : 'skos:Concept'; +} + +function termsTurtle(): string { + return LOCAL_TERMS.map(([term, kind, label, comment]) => + `mi:${term}\n a ${termType(kind)} ;\n rdfs:label ${literal(label)}@en ;\n rdfs:comment ${literal(comment)}@en ;\n rdfs:isDefinedBy <${VOCABULARY}> .`).join('\n\n'); +} + /** The whole dictionary as Turtle. `issued` maps a published path to its release date. */ export function renderTurtle(repo: RepoModel, refs: RefIndex, issued: Map = new Map()): string { const header = PREFIXES.map(([p, ns]) => `@prefix ${p}: <${ns}> .`).join('\n'); - const terms = LOCAL_TERMS.map(([term, kind, label, comment]) => - `mi:${term}\n a rdfs:${kind === 'Class' ? 'Class' : 'Property'} ;\n rdfs:label ${literal(label)}@en ;\n rdfs:comment ${literal(comment)}@en ;\n rdfs:isDefinedBy <${NS}> .`).join('\n\n'); - const entries = repo.published .filter((f) => f.doc) .sort((a, b) => a.stem.localeCompare(b.stem, 'en')) @@ -206,10 +238,65 @@ ${header} # ---------------------------------------------------------------- local terms (self-describing) -${terms} +${termsTurtle()} # ---------------------------------------------------------------- entries ${entries} `; } + +// ------------------------------------------------------------------ the mi: vocabulary at /ns (#135) + +const VOCABULARY_TITLE = 'material-identity vocabulary (mi:) — proof of concept'; +const VOCABULARY_DESCRIPTION = 'PROOF OF CONCEPT. The local terms of the material-identity dictionary and of its DPP content specifications. Not a vocabulary of the European Union or of CEN; nothing here is normative.'; + +/** The mi: terms as their own Turtle document, served at /ns for `Accept: text/turtle`. */ +export function renderVocabularyTurtle(): string { + const prefixes: Array<[string, string]> = [...PREFIXES, ['owl', 'http://www.w3.org/2002/07/owl#'], ['adms', ADMS]]; + return `# The mi: vocabulary (#135). Derived from the term list in scripts/lib/rdf.ts, like /dictionary.ttl. + +${prefixes.map(([p, ns]) => `@prefix ${p}: <${ns}> .`).join('\n')} + +<${VOCABULARY}> + a owl:Ontology ; + dcterms:title ${literal(VOCABULARY_TITLE)}@en ; + dcterms:description ${literal(VOCABULARY_DESCRIPTION)}@en ; + adms:status mi:ProofOfConcept ; + dcterms:license <${CC0}> ; + dcterms:publisher . + +${termsTurtle()} +`; +} + +/** The same vocabulary as JSON-LD (the default representation of /ns), inline context only. */ +export function renderVocabularyJsonLd(): string { + const graph: Array> = [ + { + '@id': VOCABULARY, + '@type': 'owl:Ontology', + 'dcterms:title': { '@value': VOCABULARY_TITLE, '@language': 'en' }, + 'dcterms:description': { '@value': VOCABULARY_DESCRIPTION, '@language': 'en' }, + 'adms:status': { '@id': 'mi:ProofOfConcept' }, + 'dcterms:license': { '@id': CC0 }, + 'dcterms:publisher': { '@id': 'https://material-identity.eu/' }, + }, + ...LOCAL_TERMS.map(([term, kind, label, comment]) => ({ + '@id': `mi:${term}`, + '@type': termType(kind), + 'rdfs:label': { '@value': label, '@language': 'en' }, + 'rdfs:comment': { '@value': comment, '@language': 'en' }, + 'rdfs:isDefinedBy': { '@id': VOCABULARY }, + })), + ]; + const context: Record = Object.fromEntries([ + ...PREFIXES, ['owl', 'http://www.w3.org/2002/07/owl#'], ['adms', ADMS], + ]); + return `${JSON.stringify({ '@context': context, '@graph': graph }, null, 2)}\n`; +} + +/** The term list for the HTML page — read-only view of the one source. */ +export function vocabularyTerms(): ReadonlyArray { + return LOCAL_TERMS; +} diff --git a/scripts/lib/render.ts b/scripts/lib/render.ts index f7335d7..4836f94 100644 --- a/scripts/lib/render.ts +++ b/scripts/lib/render.ts @@ -7,6 +7,9 @@ import { CANONICAL_BASE, DEF_PREFIX, type RepoFile, type RepoModel } from './repo.ts'; import { citation } from './cite.ts'; import { graphFigure } from './graph.ts'; +import type { HostedSpec, SpecRole, SpecSummary } from './spec.ts'; +import type { ReleaseContents } from './dcat.ts'; +import { releaseAsset, releaseLandingPage } from './git.ts'; export function esc(s: unknown): string { return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); @@ -40,7 +43,7 @@ export class RefIndex { const label = en(target?.preferredName) ?? (target?.shortName as string | undefined) ?? uuid; return `${esc(label)}`; } - return `${esc(s)}`; + return externalLink(s); } /** The entry that replaces this one, if any — derived, never stored. */ @@ -150,10 +153,12 @@ function pageShell( title: string, canonicalPath: string, body: string, - options: { alternateJson?: string; alternateCsl?: string; rssFeed?: boolean; nav?: string; description?: string } = {}, + options: { alternateJson?: string; alternateCsl?: string; alternateLd?: string; rssFeed?: boolean; nav?: string; description?: string } = {}, ): string { const canonicalUrl = `${CANONICAL_BASE}${canonicalPath}`; - const alternate = (options.alternateJson === undefined + const alternate = (options.alternateLd === undefined + ? '' + : `\n`) + (options.alternateJson === undefined ? '' : `\n`) + (options.alternateCsl === undefined ? '' @@ -185,7 +190,7 @@ ${body} @@ -223,8 +228,15 @@ function refHtml(uri: unknown, refs: RefIndex): string { return `${refs.link(uri)}${symbol} ${kindChip(target.objectType)}`; } -function externalLink(uri: unknown): string { - return `${esc(uri)}`; +/** + * Only an http(s) IRI becomes a link; anything else (`urn:`, `mailto:`, a `javascript:` in any + * case) is shown as escaped text. Escaping alone does not make an href safe: a spec's RDF is + * hosted, not authored, and a script IRI would run on the canonical origin (security review of #136). + */ +export const isWebIri = (uri: unknown): boolean => /^https?:\/\//i.test(String(uri)); + +function externalLink(uri: unknown, rel = 'external', label: unknown = uri): string { + return isWebIri(uri) ? `${esc(label)}` : `${esc(label)}`; } /** @@ -713,3 +725,264 @@ ${sections || '

No examples in this build.

'}`; description: 'Content specifications that use the material-identity dictionary, shown as reference examples. They are not part of the dictionary.', }); } + +// ------------------------------------------------------------------ hosted specs, catalogue, vocabulary (#135) + +/** + * The proof-of-concept banner on every page about a hosted specification, the catalogue and the + * vocabulary. In both languages the specifications themselves carry, and never dismissible. + */ +export function pocBanner(): string { + return ``; +} + +const localPath = (iri: string | undefined): string | undefined => + iri !== undefined && iri.startsWith(`${CANONICAL_BASE}/`) ? iri.slice(CANONICAL_BASE.length) : undefined; + +/** A link to an IRI: site-relative when it is ours, external otherwise; entries get their label. */ +function iriLink(iri: string | undefined, refs: RefIndex, label?: string): string { + if (iri === undefined) return ''; + if (iri.startsWith(DEF_PREFIX)) return refs.link(iri); + const local = localPath(iri); + return local === undefined ? externalLink(iri) : `${esc(label ?? iri)}`; +} + +const LICENCE_LABEL: Record = { 'http://creativecommons.org/publicdomain/zero/1.0/': 'CC0 1.0' }; + +/** One hosted specification's page: the human representation of `/spec//`. */ +export function renderSpecPage(spec: HostedSpec, summary: SpecSummary, refs: RefIndex): string { + const path = `/spec/${spec.act}/${spec.version}`; + const title = summary.title.en ?? `${spec.act} ${spec.version}`; + const otherTitles = Object.entries(summary.title).filter(([code]) => code !== 'en' && code !== '') + .sort(([a], [b]) => a.localeCompare(b, 'en')) + .map(([code, t]) => `

${esc(t)}

`).join('\n'); + // The spec check admits exactly one licence, CC0, so this shows the value it verified. + const licence = summary.license === undefined ? '' : externalLink(summary.license, 'external license', LICENCE_LABEL[summary.license] ?? summary.license); + const status = summary.status === undefined ? '' : `${iriLink(summary.status, refs, summary.status.replace(/^.*#/, 'mi:'))}`; + + const files = spec.files.map((f) => ` +${esc(f.role === 'dcat' ? 'DCAT record (merged into the catalogue)' : FILE_LABEL[f.role])} +${esc(localPath(f.url) ?? f.url)} +${esc(f.mediaType)} +${esc(f.sha256)} +`).join('\n'); + + const pins = summary.pins.map((p) => `${refs.link(p.url)}${esc(p.url)}${esc(p.sha256 ?? '')}`).join('\n'); + const members = summary.memberships.map((m) => ` +${m.path.map(esc).join('.')} +${m.entry === undefined ? '' : iriLink(m.entry, refs)} +${m.mandatory ? 'mandatory' : 'optional'}${m.condition === undefined ? '' : `
${esc(m.condition)}`} +${m.accessCategory === undefined ? '' : iriLink(m.accessCategory, refs)} +`).join('\n'); + + const body = `${pocBanner()} +

${esc(title)}

+${otherTitles} + +${row('Specification IRI', `${esc(spec.iri)}`)} +${row('Act', `${esc(spec.act)}`)} +${row('Version', `${esc(summary.version ?? spec.version)}`)} +${row('Issued', esc(summary.issued ?? ''))} +${row('Status', status)} +${row('Licence', licence)} +${row('Version series', iriLink(summary.series, refs))} +${row('Conforms to', esc(summary.conformsTo ?? ''))} +${row('DPP schema version', summary.dppSchemaVersion === undefined ? '' : `${esc(summary.dppSchemaVersion)}`)} +${row('Dictionary release', summary.releases.map((r) => releaseLink(r.iri, r.version ?? r.iri)).join(', '))} +
+${Object.keys(summary.description).length === 0 ? '' : `

Description

${langMapHtml(summary.description)}
`} + +

Files

+

Hosted byte for byte as material-identity/schema generated them, and never changed: a change is a new version at a new IRI. The SHA-256 is over the served bytes.

+
+ + +${files} + +
RepresentationURLMedia typeSHA-256
+ +

Pinned dictionary entries

+

Each pin names the bytes of one entry by digest. The build of this site checks that every pinned entry is published here and still serves exactly those bytes.

+
+ + +${pins} + +
EntryIRISHA-256
+ +

Memberships

+
+ + +${members} + +
ElementDictionary entryMandatoryAccess
`; + + return pageShell(`${title} — ${spec.version}`, path, body, { + // The canonical IRI, which serves JSON-LD by default: index.jsonld is an origin detail (code review of #136). + alternateLd: path, + description: `Proof of concept: DPP content specification ${spec.act} ${spec.version}, hosted by the material-identity dictionary. Not a content specification of the European Union.`, + }); +} + +const FILE_LABEL: Record, string> = { + spec: 'Specification (JSON-LD, SHACL)', + context: 'Passport @context', + compressed: 'JSON Schema, compressed form', + full: 'JSON Schema, full form', +}; + +/** `/spec/`: every hosted version of one act. */ +export function renderSeriesPage(act: string, versions: Array<{ spec: HostedSpec; summary: SpecSummary }>): string { + const rows = versions.map(({ spec, summary }) => ` +${esc(spec.version)} +${esc(summary.title.en ?? '')} +${esc(summary.issued ?? '')} +`).join('\n'); + const body = `${pocBanner()} +

${esc(act)} — content specifications

+

Every hosted version of the ${esc(act)} content specification. A passport always names one exact version; there is no "latest" alias to pin.

+
+ + +${rows} + +
VersionTitleIssued
+

Machine-readable: JSON-LD (DCAT dataset series) · all specifications and dictionary releases in the catalogue.

`; + return pageShell(`${act} — content specifications`, `/spec/${act}`, body, { + alternateLd: `/spec/${act}`, + description: `Proof of concept: all hosted versions of the ${act} DPP content specification.`, + }); +} + +/** `/catalog`: the DCAT catalogue as a page. */ +export function renderCatalogPage(releases: Array<{ tag: string; date: string }>, specs: Array<{ spec: HostedSpec; summary: SpecSummary }>): string { + const releaseRows = [...releases].sort((a, b) => a.tag.localeCompare(b.tag, 'en')).map((r) => ` +${esc(r.tag)} +${esc(r.date)} +`).join('\n'); + const specRows = specs.map(({ spec, summary }) => ` +${esc(summary.title.en ?? spec.iri)} +${esc(spec.act)} +${esc(spec.version)} +${esc(summary.issued ?? '')} +`).join('\n'); + const body = `${pocBanner()} +

Catalogue

+

The dictionary releases and the DPP content specifications this site hosts, as a DCAT catalogue. Machine-readable: JSON-LD.

+

Content specifications

+${specRows === '' ? '

No content specification is hosted yet.

' : `
+ + +${specRows} + +
SpecificationActVersionIssued
`} +
+

Dictionary releases

+${releaseRows === '' ? '

No release in this build.

' : `
+ + +${releaseRows} + +
ReleaseDate
`} +
`; + return pageShell('Catalogue', '/catalog', body, { + alternateLd: '/catalog', + description: 'Proof of concept: DCAT catalogue of the material-identity dictionary releases and hosted DPP content specifications.', + }); +} + +/** A release IRI as a site-relative link when it is one of ours; anything else as text, like iriLink. */ +function releaseLink(iri: string, label: string): string { + const local = localPath(iri); + return local === undefined ? `${esc(label)}` : `${esc(label)}`; +} + +/** `/release/`: one dictionary release, the entries it first shipped and everything it contains. */ +export function renderReleasePage(release: ReleaseContents, refs: RefIndex): string { + const path = `/release/${release.tag}`; + const added = release.entries.filter((e) => e.firstShipped === release.tag); + const rows = release.entries.map((e) => ` +${refs.link(e.iri)} +${esc(e.iri)} +${e.firstShipped === release.tag ? 'this release' : `${esc(e.firstShipped)}`} +${esc(e.sha256)} +`).join('\n'); + const body = `${pocBanner()} +

Release ${esc(release.tag)}

+

A tagged release of the dictionary. It is fixed once tagged: a release only ever gains successors, never changes.

+ +${row('Release IRI', `${esc(`${CANONICAL_BASE}${path}`)}`)} +${row('Version', `${esc(release.tag)}`)} +${row('Issued', esc(release.date))} +${row('Licence', externalLink('http://creativecommons.org/publicdomain/zero/1.0/', 'external license', 'CC0 1.0'))} +${row('Download', externalLink(releaseAsset(release.tag), 'external', `dictionary-${release.tag}.tar.gz`))} +${row('Release notes', externalLink(releaseLandingPage(release.tag), 'external', 'GitHub release'))} +${row('Entries', `${release.entries.length} in total, ${added.length} first shipped in this release`)} +
+

Entries

+

Every entry this release contains. The SHA-256 is over the bytes /def/<uuid> serves as JSON: the digest a content specification pinning this release names.

+${rows === '' ? '

This release contains no entry.

' : `
+ + +${rows} + +
EntryIRIFirst shippedSHA-256
`} +
+

Machine-readable: JSON-LD (DCAT dataset), served at this IRI by content negotiation · all releases: releases · catalogue.

`; + return pageShell(`Release ${release.tag}`, path, body, { + alternateLd: path, + description: `Proof of concept: material-identity dictionary release ${release.tag}, ${release.entries.length} entries.`, + }); +} + +/** `/release`: every dictionary release. */ +export function renderReleaseIndexPage(releases: ReleaseContents[]): string { + const rows = [...releases].sort((a, b) => a.tag.localeCompare(b.tag, 'en')).map((r) => ` +${esc(r.tag)} +${esc(r.date)} +${r.entries.filter((e) => e.firstShipped === r.tag).length} +${r.entries.length} +`).join('\n'); + const body = `${pocBanner()} +

Releases

+

Every tagged release of the dictionary. A content specification pins exactly one; there is no "latest" alias to pin.

+${rows === '' ? '

No release in this build.

' : `
+ + +${rows} + +
ReleaseIssuedFirst shippedEntries
`} +

Machine-readable: JSON-LD (DCAT catalogue) · everything in the catalogue.

`; + return pageShell('Releases', '/release', body, { + alternateLd: '/release', + description: 'Proof of concept: every release of the material-identity dictionary.', + }); +} + +/** `/ns`: the mi: vocabulary, one anchor per term so `…/ns#` lands on its definition. */ +export function renderVocabularyPage(terms: ReadonlyArray): string { + const rows = terms.map(([term, kind, label, comment]) => ` +mi:${esc(term)} +${esc(kind)} +${esc(label)} +${esc(comment)} +`).join('\n'); + const body = `${pocBanner()} +

Vocabulary

+

The local terms of this dictionary and of the content specifications it hosts, in the namespace https://material-identity.eu/ns#. Standard vocabularies (SKOS, DCTerms, DCAT, SHACL, QUDT) are used wherever one fits; a term is defined here only when none does.

+

Machine-readable: Turtle · JSON-LD. Both are also served at /ns itself, by content negotiation.

+
+ + +${rows} + +
TermKindLabelDefinition
`; + return pageShell('Vocabulary', '/ns', body, { + alternateLd: '/ns', + description: 'Proof of concept: the mi: vocabulary of the material-identity dictionary and its DPP content specifications.', + }); +} diff --git a/scripts/lib/spec.ts b/scripts/lib/spec.ts new file mode 100644 index 0000000..b97a1bd --- /dev/null +++ b/scripts/lib/spec.ts @@ -0,0 +1,358 @@ +/** + * Hosted DPP content specifications (#135; schema repo docs/design/content-spec-jsonld.md). + * + * material-identity/schema GENERATES a specification; this repo only HOSTS it, byte-identical. + * The source tree is `spec///`, add-only like published/ (check 1), holding exactly + * the files the hand-over carries. The build copies them under the names the Worker maps the + * canonical IRIs onto, so nothing here ever rewrites a byte. + * + * Reading is through oxigraph — already a dependency for the Turtle tests — with no document + * loader: a remote `@context` fails to load, which is exactly the design's rule that a spec's own + * context is inline (§2.3), so the digest covers its complete meaning. + */ +import { createHash } from 'node:crypto'; +import { lstatSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import oxigraph from 'oxigraph'; +import { CANONICAL_BASE, type ValidationIssue } from './repo.ts'; + +export const SPEC_ROOT = 'spec'; +export const ACT_PATTERN = /^[a-z][a-z0-9-]*$/; +export const VERSION_PATTERN = /^v\d+\.\d+\.\d+$/; + +const SH = 'http://www.w3.org/ns/shacl#'; +const MI = 'https://material-identity.eu/ns#'; +const DCAT = 'http://www.w3.org/ns/dcat#'; +const DCTERMS = 'http://purl.org/dc/terms/'; +const SPDX = 'http://spdx.org/rdf/terms#'; +const ADMS = 'http://www.w3.org/ns/adms#'; + +export const PREFIXES = `PREFIX sh: <${SH}> PREFIX mi: <${MI}> PREFIX dcat: <${DCAT}> PREFIX dcterms: <${DCTERMS}> PREFIX spdx: <${SPDX}> PREFIX adms: <${ADMS}> `; + +export type SpecRole = 'spec' | 'context' | 'dcat' | 'compressed' | 'full'; + +/** One handed-over file and, if it is served on its own, the URL and origin name it is served as. */ +export interface SpecFile { + role: SpecRole; + /** Name in the hand-over and in `spec///`. */ + source: string; + /** Name below `site/spec///`; undefined for the DCAT record, which /catalog serves. */ + served?: string; + /** Canonical URL; for the DCAT record, the catalogue it is merged into. */ + url: string; + mediaType: string; + bytes: Buffer; + sha256: string; +} + +export interface HostedSpec { + act: string; + version: string; + /** `https://material-identity.eu/spec//` */ + iri: string; + /** `spec//` */ + relDir: string; + files: SpecFile[]; + /** The specification's RDF (its own document only). */ + graph: oxigraph.Store; + /** The DCAT record as JSON (merged into /catalog as written) and as RDF (checked). */ + dcatJson: Record; + dcatGraph: oxigraph.Store; +} + +export interface SpecLoad { + specs: HostedSpec[]; + issues: ValidationIssue[]; +} + +export const specIri = (act: string, version: string): string => `${CANONICAL_BASE}/spec/${act}/${version}`; + +/** The hand-over, file by file (design §7.2 step 3), and where each one is served. */ +function layout(act: string, iri: string): Array> { + return [ + { role: 'spec', source: `${act}.jsonld`, served: 'index.jsonld', url: iri, mediaType: 'application/ld+json' }, + { role: 'context', source: `${act}.context.jsonld`, served: 'context.jsonld', url: `${iri}/context.jsonld`, mediaType: 'application/ld+json' }, + { role: 'dcat', source: `${act}.dcat.jsonld`, url: `${CANONICAL_BASE}/catalog`, mediaType: 'application/ld+json' }, + { role: 'compressed', source: 'compressed.schema.json', served: 'compressed.schema.json', url: `${iri}/compressed.schema.json`, mediaType: 'application/schema+json' }, + { role: 'full', source: 'full.schema.json', served: 'full.schema.json', url: `${iri}/full.schema.json`, mediaType: 'application/schema+json' }, + ]; +} + +/** Parse JSON-LD with no document loader: remote contexts fail, as they must (design §2.3). */ +export function parseJsonLd(text: string, base: string): oxigraph.Store { + const store = new oxigraph.Store(); + store.load(text, { format: 'application/ld+json', base_iri: base }); + return store; +} + +const dirs = (abs: string): string[] => readdirSync(abs).sort(); + +/** + * What an entry under spec/ is, WITHOUT following links. A symlink is refused outright: spec/ is + * add-only (check 1), and a link would let a later change outside spec/ alter what an immutable + * IRI serves — or, absolute, publish a file of the build machine (security review of #136). + */ +type EntryKind = 'dir' | 'file' | 'missing' | 'other'; +function entryKind(abs: string): EntryKind { + let st; + try { + st = lstatSync(abs); + } catch { + return 'missing'; + } + return st.isDirectory() ? 'dir' : st.isFile() ? 'file' : 'other'; +} +const NOT_REGULAR = 'is a symlink or other special file — everything under spec/ must be a regular file or directory'; + +/** + * Every `spec///`, sorted by act then version. Structural problems (names, missing or + * extra files, unparseable JSON or JSON-LD) are returned as issues and the directory is skipped. + */ +export function loadSpecs(root: string): SpecLoad { + const specs: HostedSpec[] = []; + const issues: ValidationIssue[] = []; + const check = 'spec layout'; + const base = join(root, SPEC_ROOT); + const baseKind = entryKind(base); + if (baseKind === 'missing') return { specs, issues }; + if (baseKind !== 'dir') return { specs, issues: [{ check, file: SPEC_ROOT, message: NOT_REGULAR }] }; + + for (const act of dirs(base)) { + const actAbs = join(base, act); + const actKind = entryKind(actAbs); + if (actKind === 'other') { + issues.push({ check, file: `${SPEC_ROOT}/${act}`, message: NOT_REGULAR }); + continue; + } + if (actKind !== 'dir' || !ACT_PATTERN.test(act)) { + issues.push({ check, file: `${SPEC_ROOT}/${act}`, message: `not an act directory (expected ${ACT_PATTERN})` }); + continue; + } + for (const version of dirs(actAbs).sort(compareVersions)) { + const relDir = `${SPEC_ROOT}/${act}/${version}`; + const abs = join(actAbs, version); + const versionKind = entryKind(abs); + if (versionKind === 'other') { + issues.push({ check, file: relDir, message: NOT_REGULAR }); + continue; + } + if (versionKind !== 'dir' || !VERSION_PATTERN.test(version)) { + issues.push({ check, file: relDir, message: `not a version directory (expected ${VERSION_PATTERN})` }); + continue; + } + const iri = specIri(act, version); + const expected = layout(act, iri); + const present = new Set(dirs(abs)); + const before = issues.length; + for (const f of expected) { + if (!present.has(f.source)) issues.push({ check, file: `${relDir}/${f.source}`, message: 'missing from the hand-over' }); + } + for (const name of present) { + const kind = entryKind(join(abs, name)); + if (kind === 'other') issues.push({ check, file: `${relDir}/${name}`, message: NOT_REGULAR }); + else if (kind !== 'file' || !expected.some((f) => f.source === name)) issues.push({ check, file: `${relDir}/${name}`, message: 'not part of a content-specification hand-over' }); + } + if (issues.length > before) continue; + + const files: SpecFile[] = expected.map((f) => { + const bytes = readFileSync(join(abs, f.source)); + return { ...f, bytes, sha256: createHash('sha256').update(bytes).digest('hex') }; + }); + const parsed = new Map(); + for (const f of files) { + const text = f.bytes.toString('utf8'); + try { + const json: unknown = JSON.parse(text); + parsed.set(f.role, { json, graph: f.source.endsWith('.jsonld') ? parseJsonLd(text, f.url) : undefined }); + } catch (err) { + issues.push({ check, file: `${relDir}/${f.source}`, message: `does not parse as ${f.source.endsWith('.jsonld') ? 'JSON-LD' : 'JSON'}: ${(err as Error).message}` }); + } + } + if (issues.length > before) continue; + const dcatJson = parsed.get('dcat')!.json; + if (!dcatJson || typeof dcatJson !== 'object' || Array.isArray(dcatJson)) { + issues.push({ check, file: `${relDir}/${act}.dcat.jsonld`, message: 'the DCAT record is not a JSON object' }); + continue; + } + specs.push({ + act, version, iri, relDir, files, + graph: parsed.get('spec')!.graph!, + dcatJson: dcatJson as Record, + dcatGraph: parsed.get('dcat')!.graph!, + }); + } + } + return { specs, issues }; +} + +/** `v0.0.10` after `v0.0.9`: numeric, not lexical. Non-versions sort after, by name. */ +export function compareVersions(a: string, b: string): number { + const parts = (v: string): number[] | undefined => (VERSION_PATTERN.test(v) ? v.slice(1).split('.').map(Number) : undefined); + const pa = parts(a); + const pb = parts(b); + if (!pa || !pb) return pa ? -1 : pb ? 1 : a.localeCompare(b, 'en'); + for (let i = 0; i < 3; i++) if (pa[i] !== pb[i]) return pa[i] - pb[i]; + return 0; +} + +// ------------------------------------------------------------------ reading a spec + +type Value = oxigraph.NamedNode | oxigraph.BlankNode | oxigraph.Literal; +type Row = Map; + +export function select(store: oxigraph.Store, where: string): Row[] { + return store.query(`${PREFIXES} SELECT * WHERE { ${where} }`) as unknown as Row[]; +} + +export function ask(store: oxigraph.Store, where: string): boolean { + return store.query(`${PREFIXES} ASK { ${where} }`) as unknown as boolean; +} + +/** Literal values of one predicate on one subject, keyed by language tag ('' when untagged). */ +function langValues(store: oxigraph.Store, subject: string, predicate: string): Record { + const out: Record = {}; + const rows = select(store, `<${subject}> ${predicate} ?v`).sort((a, b) => a.get('v')!.value.localeCompare(b.get('v')!.value, 'en')); + for (const row of rows) { + const v = row.get('v')!; + const key = v.termType === 'Literal' ? (v as oxigraph.Literal).language : ''; + if (out[key] === undefined) out[key] = v.value; + } + return out; +} + +function one(store: oxigraph.Store, subject: string, predicate: string): string | undefined { + return select(store, `<${subject}> ${predicate} ?v`).map((r) => r.get('v')!.value).sort()[0]; +} + +export interface Pin { + /** `dcat:downloadURL` — the pinned entry IRI. */ + url: string; + sha256?: string; +} + +/** + * `dcterms:requires` → distributions (pinned entries) of the subject: one Pin per distinct + * (entry, checksum) pair, sorted by entry then checksum. Every checksum stated for an entry is kept, + * so a second, conflicting digest is checked like the first instead of whichever SPARQL returned + * first (code review of #136); a distribution without one appears with `sha256` undefined. + */ +export function distributionPins(store: oxigraph.Store, subject: string, predicate = 'dcterms:requires'): Pin[] { + const rows = select(store, `<${subject}> ${predicate} ?d . ?d dcat:downloadURL ?u . OPTIONAL { ?d spdx:checksum ?c . ?c spdx:checksumValue ?sum }`); + const pins = new Map(); + for (const row of rows) { + const url = row.get('u')!.value; + const sha256 = row.get('sum')?.value; + pins.set(`${url} ${sha256 ?? ''}`, { url, sha256 }); + } + return [...pins.values()].sort((a, b) => a.url.localeCompare(b.url, 'en') || (a.sha256 ?? '').localeCompare(b.sha256 ?? '', 'en')); +} + +/** + * Every dictionary reference the spec's memberships make — `sh:path` (when an IRI), `mi:itemType` + * and `mi:accessCategory` of each shape carrying an `mi:elementId` — sorted and distinct. The same + * set the schema repo's publication gate requires to equal the pins (JsonLdGates: "every reference + * is pinned, every pin is used"). + */ +export function membershipReferences(store: oxigraph.Store): string[] { + const rows = select(store, `?p mi:elementId ?id . { ?p sh:path ?r } UNION { ?p mi:itemType ?r } UNION { ?p mi:accessCategory ?r } FILTER(isIRI(?r))`); + return [...new Set(rows.map((r) => r.get('r')!.value))].sort(); +} + +export interface Membership { + /** Dotted element-id path from the root, e.g. `nominalDimensions.nominalThickness`. */ + path: string[]; + /** The dictionary entry: `sh:path` when it is an IRI, else the list item's `mi:itemType`. */ + entry?: string; + mandatory: boolean; + accessCategory?: string; + condition?: string; +} + +export interface SpecSummary { + title: Record; + description: Record; + version?: string; + issued?: string; + license?: string; + status?: string; + series?: string; + conformsTo?: string; + dppSchemaVersion?: string; + passportContext?: string; + releases: Array<{ iri: string; version?: string }>; + pins: Pin[]; + memberships: Membership[]; +} + +/** What the HTML, the series index and the catalogue say about a spec — read from its RDF. */ +export function summarize(spec: HostedSpec): SpecSummary { + const g = spec.graph; + const s = spec.iri; + const releases = select(g, `<${s}> dcterms:requires ?r . ?r a dcat:Dataset . OPTIONAL { ?r dcat:version ?v }`) + .map((r) => ({ iri: r.get('r')!.value, version: r.get('v')?.value })) + .sort((a, b) => a.iri.localeCompare(b.iri, 'en')); + return { + title: langValues(g, s, 'dcterms:title'), + description: langValues(g, s, 'dcterms:description'), + version: one(g, s, 'dcat:version'), + issued: one(g, s, 'dcterms:issued'), + license: one(g, s, 'dcterms:license'), + status: one(g, s, 'adms:status'), + series: one(g, s, 'dcat:isVersionOf'), + conformsTo: one(g, s, 'dcterms:conformsTo'), + dppSchemaVersion: one(g, s, 'mi:dppSchemaVersion'), + passportContext: one(g, s, 'mi:passportContext'), + releases: releases.filter((r, i) => i === 0 || r.iri !== releases[i - 1].iri), + pins: distributionPins(g, s), + memberships: memberships(g, s), + }; +} + +/** + * Memberships depth-first in `sh:order` (design §3.2: array position is not significant in RDF). + * A membership is any property shape with an `mi:elementId`; its children sit under its `sh:node`. + */ +function memberships(g: oxigraph.Store, root: string): Membership[] { + const rows = select(g, ` + { <${root}> sh:property ?p . BIND(<${root}> AS ?parent) } UNION { ?parent sh:node ?n . ?n sh:property ?p } + ?p mi:elementId ?id . + OPTIONAL { ?p sh:path ?path FILTER(isIRI(?path)) } + OPTIONAL { ?p mi:itemType ?item } + OPTIONAL { ?p sh:order ?order } + OPTIONAL { ?p sh:minCount ?min } + OPTIONAL { ?p mi:accessCategory ?access } + OPTIONAL { ?p mi:condition ?cond FILTER(langMatches(lang(?cond), "en")) }`); + const key = (t: Value): string => `${t.termType}:${t.value}`; + interface Node { key: string; id: string; order: number; m: Omit } + const children = new Map(); + const seen = new Set(); + for (const r of rows) { + const k = key(r.get('p')!); + if (seen.has(k)) continue; + seen.add(k); + const parent = key(r.get('parent')!); + const node: Node = { + key: k, + id: r.get('id')!.value, + order: Number(r.get('order')?.value ?? Number.MAX_SAFE_INTEGER), + m: { + entry: r.get('path')?.value ?? r.get('item')?.value, + mandatory: Number(r.get('min')?.value ?? 0) >= 1, + accessCategory: r.get('access')?.value, + condition: r.get('cond')?.value, + }, + }; + children.set(parent, [...(children.get(parent) ?? []), node]); + } + const out: Membership[] = []; + const walk = (parentKey: string, path: string[], visiting: Set): void => { + const list = [...(children.get(parentKey) ?? [])].sort((a, b) => a.order - b.order || a.id.localeCompare(b.id, 'en')); + for (const n of list) { + if (visiting.has(n.key)) continue; // a shape graph may be cyclic; a table may not + out.push({ path: [...path, n.id], ...n.m }); + walk(n.key, [...path, n.id], new Set([...visiting, n.key])); + } + }; + walk(`NamedNode:${root}`, [], new Set()); + return out; +} diff --git a/scripts/lib/styles.css b/scripts/lib/styles.css index 1767098..575ac4b 100644 --- a/scripts/lib/styles.css +++ b/scripts/lib/styles.css @@ -19,6 +19,7 @@ --chip-neutral-bg: #e7efea; --chip-neutral-fg: #2f4a3b; --chip-tier-bg: #ffedd5; --chip-tier-fg: #c2410c; --dead-bg: #fbe4e4; --dead-edge: #b3423d; + --poc-bg: #fef3c7; --poc-edge: #b45309; } @media (prefers-color-scheme: dark) { :root { @@ -38,6 +39,7 @@ --chip-neutral-bg: #1c3529; --chip-neutral-fg: #d2e3d9; --chip-tier-bg: #6a300d; --chip-tier-fg: #fed7aa; --dead-bg: #4a1d1b; --dead-edge: #e07a75; + --poc-bg: #4a3208; --poc-edge: #f0b44c; } } @@ -205,3 +207,7 @@ ul.graph-key li { display: flex; align-items: center; gap: 0.45rem; } } @media (prefers-reduced-motion: no-preference) { a { transition: text-decoration-thickness 120ms; } } @media print { .mast, footer, .btn { display: none; } } + +/* Proof-of-concept banner (#135): on every page about a hosted specification, the catalogue and /ns. */ +.banner.poc { background: var(--poc-bg); border-color: var(--poc-edge); } +.banner.poc p { margin: 0.25rem 0; } diff --git a/test/checks-m2.test.ts b/test/checks-m2.test.ts index 53e6fa6..674385a 100644 --- a/test/checks-m2.test.ts +++ b/test/checks-m2.test.ts @@ -7,7 +7,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { loadRepo } from '../scripts/lib/repo.ts'; import { checkImmutability, checkMovePurity } from '../scripts/lib/checks.ts'; -import type { GitContext } from '../scripts/lib/git.ts'; +import { type GitContext, gitEnv } from '../scripts/lib/git.ts'; import { runValidation } from '../scripts/validate.ts'; const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures'); @@ -65,7 +65,7 @@ test('check 6 — pure move (only id differs) passes; edited move fails; withdra // ---------------------------------------------------------------- integration: real git repo const gitc = (cwd: string, ...args: string[]) => - execFileSync('git', ['-C', cwd, '-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', '-c', 'tag.gpgsign=false', ...args], { stdio: 'pipe' }); + execFileSync('git', ['-C', cwd, '-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', '-c', 'tag.gpgsign=false', ...args], { stdio: 'pipe', env: gitEnv() }); test('integration — temp git repo: pure publish move green; edited move, tamper, delete red', () => { const root = mkdtempSync(join(tmpdir(), 'dict-validate-')); diff --git a/test/checks.test.ts b/test/checks.test.ts index aeb83cd..83fd786 100644 --- a/test/checks.test.ts +++ b/test/checks.test.ts @@ -90,7 +90,7 @@ test('check 5 — isDefinedBy and replaces are exempt from pinning', () => { test('runChecks aggregates load errors and all checks 1–6', () => { const results = runChecks(load('red-yaml')); - assert.equal(results.length, 9); + assert.equal(results.length, 11); const loadResult = results.find((r) => r.name === 'load'); assert.ok(loadResult && loadResult.issues.length === 1); const failing = results.filter((r) => r.issues.length > 0); diff --git a/test/feed.test.ts b/test/feed.test.ts index 21d7413..b85cd18 100644 --- a/test/feed.test.ts +++ b/test/feed.test.ts @@ -7,7 +7,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { DEF_PREFIX, type RepoModel } from '../scripts/lib/repo.ts'; import { renderFeed } from '../scripts/lib/feed.ts'; -import { getAddedDates } from '../scripts/lib/git.ts'; +import { getAddedDates, gitEnv } from '../scripts/lib/git.ts'; const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures'); @@ -96,7 +96,7 @@ test('renderFeed: title/description text is XML-escaped', () => { // ------------------------------------------------------------------ getAddedDates (#56) const gitc = (cwd: string, ...args: string[]) => - execFileSync('git', ['-C', cwd, '-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', ...args], { stdio: 'pipe' }); + execFileSync('git', ['-C', cwd, '-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', ...args], { stdio: 'pipe', env: gitEnv() }); test('getAddedDates: finds the real add-date, and specifically the *original* one across a draft -> published rename', () => { const root = mkdtempSync(join(tmpdir(), 'dict-feed-')); @@ -108,7 +108,7 @@ test('getAddedDates: finds the real add-date, and specifically the *original* on gitc(root, 'commit', '-m', 'draft', '--date=2026-01-01T00:00:00'); mkdirSync(join(root, 'published')); - execFileSync('git', ['-C', root, 'mv', 'drafts/x.yaml', 'published/x.yaml']); + execFileSync('git', ['-C', root, 'mv', 'drafts/x.yaml', 'published/x.yaml'], { env: gitEnv() }); gitc(root, 'commit', '-m', 'publish (git sees this as a rename, not an add)', '--date=2026-06-01T00:00:00'); // a later, unrelated modification must not shift the recorded add-date diff --git a/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json b/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json new file mode 100644 index 0000000..4dab70b --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://material-identity.eu/spec/demo/v0.0.1/compressed.schema.json", + "$comment": "SPDX-License-Identifier: CC0-1.0. PROOF OF CONCEPT test fixture.", + "type": "object", + "required": ["mechanicalProperties"], + "properties": { + "mechanicalProperties": { "type": "object", "required": ["maxPressure"], "properties": { "maxPressure": { "type": "number" } }, "additionalProperties": false }, + "steelmakingRoute": { "type": "string", "format": "uri" } + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld new file mode 100644 index 0000000..db597f0 --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld @@ -0,0 +1,16 @@ +{ + "@context": { + "@version": 1.1, + "@protected": true, + "def": "https://material-identity.eu/def/", + "mi": "https://material-identity.eu/ns#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "contentSpecificationIds": { "@id": "@type", "@container": "@set" }, + "dppSchemaVersion": { "@id": "mi:dppSchemaVersion", "@type": "@id" }, + "mechanicalProperties": { + "@id": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", + "@context": { "maxPressure": { "@id": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", "@type": "xsd:float" } } + }, + "steelmakingRoute": { "@id": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", "@type": "@id" } + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld new file mode 100644 index 0000000..d896773 --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld @@ -0,0 +1,56 @@ +{ + "@context": { + "dcat": "http://www.w3.org/ns/dcat#", + "dcterms": "http://purl.org/dc/terms/", + "adms": "http://www.w3.org/ns/adms#", + "spdx": "http://spdx.org/rdf/terms#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "mi": "https://material-identity.eu/ns#" + }, + "@id": "https://material-identity.eu/catalog", + "@type": "dcat:Catalog", + "dcat:dataset": { + "@id": "https://material-identity.eu/spec/demo/v0.0.1", + "@type": "dcat:Dataset", + "dcterms:title": { "@value": "Demo content specification — test fixture (PoC)", "@language": "en" }, + "dcterms:description": { "@value": "PROOF OF CONCEPT. Not a content specification of the European Union.", "@language": "en" }, + "adms:status": { "@id": "mi:ProofOfConcept" }, + "dcat:version": "v0.0.1", + "dcat:isVersionOf": { "@id": "https://material-identity.eu/spec/demo" }, + "dcterms:issued": { "@value": "2026-11-02", "@type": "xsd:date" }, + "dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }, + "dcterms:conformsTo": { "@id": "urn:cen:en:18223:2026" }, + "dcterms:requires": { "@id": "https://material-identity.eu/release/v2026.08.28" }, + "dcat:distribution": [ + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#jsonld", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1" }, + "dcat:mediaType": "application/ld+json", + "dcterms:conformsTo": { "@id": "http://www.w3.org/ns/shacl#" }, + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "c90dfd49a794e177b19c1559427c1c504ba58016c49b9a01ef353565f86a4bd7" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#context", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/context.jsonld" }, + "dcat:mediaType": "application/ld+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "64c71906fd2eeb51a2e6b54480dc9ac530ceed17f949dd483da3ea7c8d6f97b4" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#compressed", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/compressed.schema.json" }, + "dcat:mediaType": "application/schema+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "eb16cf4331e090f6813d873790ff70a3445afbd2ea32e4de5abdbe0ff35546e4" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#full", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/full.schema.json" }, + "dcat:mediaType": "application/schema+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "bb27ef5ee80e4abcc1f38a6e1a09df0d4e57185ae73fc8b793a7a5dccad93a1d" } + } + ] + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld new file mode 100644 index 0000000..50fdcde --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld @@ -0,0 +1,90 @@ +{ + "@context": { + "@version": 1.1, + "sh": "http://www.w3.org/ns/shacl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dcterms": "http://purl.org/dc/terms/", + "dcat": "http://www.w3.org/ns/dcat#", + "adms": "http://www.w3.org/ns/adms#", + "spdx": "http://spdx.org/rdf/terms#", + "mi": "https://material-identity.eu/ns#", + "def": "https://material-identity.eu/def/", + "s": "https://material-identity.eu/spec/demo/v0.0.1#", + "title": { "@id": "dcterms:title", "@container": "@language" }, + "description": { "@id": "dcterms:description", "@container": "@language" }, + "pin": { "@id": "dcterms:requires", "@container": "@set" }, + "entry": { "@id": "dcat:downloadURL", "@type": "@id" }, + "checksum": "spdx:checksum", + "algorithm": { "@id": "spdx:algorithm", "@type": "@id" }, + "sha256": "spdx:checksumValue", + "property": { "@id": "sh:property", "@container": "@set" }, + "path": { "@id": "sh:path", "@type": "@id" }, + "datatype": { "@id": "sh:datatype", "@type": "@id" }, + "nodeKind": { "@id": "sh:nodeKind", "@type": "@id" }, + "minCount": "sh:minCount", + "maxCount": "sh:maxCount", + "order": "sh:order", + "node": "sh:node", + "closed": "sh:closed", + "elementId": "mi:elementId", + "accessCategory": { "@id": "mi:accessCategory", "@type": "@vocab" }, + "granularity": { "@id": "mi:granularity", "@type": "@id" }, + "condition": { "@id": "mi:condition", "@container": "@language" }, + "authorityOnly": "https://material-identity.eu/def/7a1e2c3d-4b5f-4a6e-9c8d-1f2e3d4c5b6a" + }, + "@id": "https://material-identity.eu/spec/demo/v0.0.1", + "@type": ["mi:ContentSpecification", "sh:NodeShape", "rdfs:Class"], + "title": { + "en": "Demo content specification — test fixture (PoC)", + "de": "Demo-Inhaltsspezifikation — Testdatensatz (PoC)" + }, + "description": { + "en": "PROOF OF CONCEPT. A test fixture of the dictionary repository. Not a content specification of the European Union.", + "de": "PROOF OF CONCEPT. Ein Testdatensatz des Wörterbuch-Repositorys. Keine Inhaltsspezifikation der Europäischen Union." + }, + "adms:status": { "@id": "mi:ProofOfConcept" }, + "dcat:version": "v0.0.1", + "dcat:isVersionOf": { "@id": "https://material-identity.eu/spec/demo" }, + "dcterms:issued": { "@value": "2026-11-02", "@type": "xsd:date" }, + "dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }, + "dcterms:rights": "SPDX-License-Identifier: CC0-1.0", + "dcterms:publisher": { "@id": "https://material-identity.eu/" }, + "dcterms:conformsTo": { "@id": "urn:cen:en:18223:2026" }, + "mi:act": "demo", + "mi:dppSchemaVersion": { "@id": "https://material-identity.org/schemas/dpp/v0.0.3" }, + "mi:passportContext": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/context.jsonld" }, + + "pin": [ + { "@id": "https://material-identity.eu/release/v2026.08.28", "@type": "dcat:Dataset", "dcat:version": "v2026.08.28" }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "cb2f32484b2602aad2a2fbb189722ba140e310927745de7a00e6ff8aefd0722d" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "51a28c90bcf24fc116251ef96216a331f278556bf92935dda49f345971cfe075" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "f136650e084da6b9e55657d0556957f14bcf8b1aed7256fb3af4ba46f7f94476" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:7a1e2c3d-4b5f-4a6e-9c8d-1f2e3d4c5b6a", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "a9ba77a780962cfe6242caedf9d8a33919096d15b234a7a15629269c3c3a6908" } } + ], + + "closed": true, + "sh:ignoredProperties": { "@list": [{ "@id": "rdf:type" }] }, + "property": [ + { "path": "mi:dppSchemaVersion", "sh:hasValue": { "@id": "https://material-identity.org/schemas/dpp/v0.0.3" }, "minCount": 1, "maxCount": 1 }, + + { "@id": "s:steelmakingRoute", "elementId": "steelmakingRoute", "order": 2, + "path": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", "nodeKind": "sh:IRI", "maxCount": 1, + "accessCategory": "authorityOnly", "granularity": "mi:batch", + "condition": { "en": "Mandatory where the route is known.", "de": "Verpflichtend, wenn die Route bekannt ist." } }, + + { "@id": "s:mechanicalProperties", "elementId": "mechanicalProperties", "order": 1, + "path": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", "minCount": 1, "maxCount": 1, + "accessCategory": "authorityOnly", "granularity": "mi:batch", + "node": { "@id": "s:MechanicalProperties", "@type": "sh:NodeShape", "closed": true, "property": [ + { "@id": "s:mechanicalProperties.maxPressure", "elementId": "maxPressure", "order": 1, + "path": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", "datatype": "xsd:float", "minCount": 1, "maxCount": 1, + "accessCategory": "authorityOnly" } + ] } } + ] +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/full.schema.json b/test/fixtures/green/spec/demo/v0.0.1/full.schema.json new file mode 100644 index 0000000..dae232c --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/full.schema.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://material-identity.eu/spec/demo/v0.0.1/full.schema.json", + "$comment": "SPDX-License-Identifier: CC0-1.0. PROOF OF CONCEPT test fixture.", + "type": "object", + "required": ["elements"], + "properties": { "elements": { "type": "array", "maxItems": 2 } } +} diff --git a/test/index-worker.test.ts b/test/index-worker.test.ts index c2b7349..7a725f4 100644 --- a/test/index-worker.test.ts +++ b/test/index-worker.test.ts @@ -173,3 +173,148 @@ test('worker fetch handler maps to the origin and stamps headers (stubbed origin globalThis.fetch = realFetch; } }); + +// ------------------------------------------------------------------ content specifications, catalogue, vocabulary (#135) + +test('decide: a spec IRI is JSON-LD by default (immutable), HTML for browsers (short cache), cite-as itself', () => { + const ld = decide('/spec/steel/v0.0.3', 'application/ld+json'); + assert.deepEqual(ld, { + originPath: '/spec/steel/v0.0.3/index.jsonld', + headers: { + 'content-type': 'application/ld+json', + 'cache-control': 'public, max-age=31536000, immutable', + vary: 'Accept', + link: '; rel="cite-as"', + }, + }); + assert.equal(decide('/spec/steel/v0.0.3', '*/*').originPath, '/spec/steel/v0.0.3/index.jsonld'); + assert.equal(decide('/spec/steel/v0.0.3', null).originPath, '/spec/steel/v0.0.3/index.jsonld'); + + const html = decide('/spec/steel/v0.0.3', 'text/html,application/xhtml+xml,*/*;q=0.8'); + assert.equal(html.originPath, '/spec/steel/v0.0.3/index.html'); + assert.equal(html.headers['content-type'], 'text/html; charset=utf-8'); + // the HTML is this site's rendering, not the hand-over: banner or footer changes must reach readers + assert.equal(html.headers['cache-control'], 'public, max-age=300'); + assert.equal(html.headers.vary, 'Accept'); + assert.equal(decide('/spec/steel/v0.0.3', '*/*').headers['cache-control'], 'public, max-age=31536000, immutable'); + assert.equal(html.headers.link, '; rel="canonical", ; rel="cite-as"'); + // multi-digit versions and hyphenated acts are legitimate + assert.equal(decide('/spec/construction-products/v10.20.300', '*/*').originPath, '/spec/construction-products/v10.20.300/index.jsonld'); +}); + +test('decide: a spec\'s sub-resources are typed and immutable, whatever the Accept header', () => { + const ctx = decide('/spec/steel/v0.0.3/context.jsonld', 'text/html'); + assert.deepEqual(ctx, { originPath: '/spec/steel/v0.0.3/context.jsonld', headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=31536000, immutable' } }); + for (const name of ['compressed.schema.json', 'full.schema.json']) { + const schema = decide(`/spec/steel/v0.0.3/${name}`, '*/*'); + assert.deepEqual(schema, { originPath: `/spec/steel/v0.0.3/${name}`, headers: { 'content-type': 'application/schema+json', 'cache-control': 'public, max-age=31536000, immutable' } }); + } +}); + +test('decide: the series index and the catalogue negotiate like a spec but keep a short cache', () => { + for (const [path, dir] of [['/spec/steel', '/spec/steel'], ['/catalog', '/catalog']]) { + assert.deepEqual(decide(path, '*/*'), { originPath: `${dir}/index.jsonld`, headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.deepEqual(decide(path, 'text/html'), { originPath: `${dir}/index.html`, headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.ok(!decide(path, '*/*').headers['cache-control'].includes('immutable'), `${path} gains members, so it must never be immutable`); + } +}); + +test('decide: /ns serves the vocabulary as JSON-LD, Turtle on request, HTML for browsers', () => { + assert.deepEqual(decide('/ns', '*/*'), { originPath: '/ns/index.jsonld', headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.deepEqual(decide('/ns', 'text/turtle'), { originPath: '/ns/index.ttl', headers: { 'content-type': 'text/turtle; charset=utf-8', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.equal(decide('/ns', 'text/html').originPath, '/ns/index.html'); + assert.equal(decide('/ns', 'text/html, text/turtle;q=0.5').originPath, '/ns/index.html'); // a browser always gets the page +}); + +test('decide: anything that is not exactly a spec shape falls through as before', () => { + for (const p of [ + '/spec', '/spec/', '/spec/steel/', '/spec/Steel', '/spec/1steel', '/spec/steel_x', '/spec/steel/v0.0.3/', '/spec/steel/0.0.3', + '/spec/steel/v0.3', '/spec/steel/latest', '/spec/steel/v0.0.3/steel.dcat.jsonld', '/spec/steel/v0.0.3/index.jsonld', + '/spec/steel/v0.0.3/context.jsonld/x', '/spec/a/b/c/d', '/catalog/', '/catalog/index.jsonld', '/ns/', '/ns/index.ttl', '/nsx', + ]) { + assert.deepEqual(decide(p, 'text/html'), { originPath: p, headers: { 'cache-control': 'public, max-age=300' } }, `${p} must pass through`); + } + // no origin path we map to may start with a dot (#110) + for (const p of ['/spec/steel/v0.0.3', '/spec/steel/v0.0.3/context.jsonld', '/spec/steel', '/catalog', '/ns']) { + for (const accept of ['*/*', 'text/html', 'text/turtle']) assert.ok(!decide(p, accept).originPath.includes('/.')); + } +}); + +test('worker fetch handler serves a spec as JSON-LD from its origin file (stubbed origin)', async () => { + const seen: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + seen.push(String(input)); + return new Response('{"@id":"x"}', { status: 200, headers: { 'content-type': 'application/octet-stream' } }); + }) as typeof fetch; + try { + const res = await worker.fetch(new Request('https://material-identity.eu/spec/steel/v0.0.3'), { ORIGIN: 'https://origin.example/dictionary' }); + assert.equal(res.status, 200); + assert.equal(res.headers.get('content-type'), 'application/ld+json'); // the origin's guess is overridden + assert.equal(res.headers.get('vary'), 'Accept'); + assert.deepEqual(seen, ['https://origin.example/dictionary/spec/steel/v0.0.3/index.jsonld']); + } finally { + globalThis.fetch = realFetch; + } +}); + +// ------------------------------------------------------------------ dictionary releases (#135) + +test('decide: a release IRI is JSON-LD by default (immutable), HTML for browsers (short cache), cite-as itself', () => { + assert.deepEqual(decide('/release/v2026.10.01', 'application/ld+json'), { + originPath: '/release/v2026.10.01/index.jsonld', + headers: { + 'content-type': 'application/ld+json', + 'cache-control': 'public, max-age=31536000, immutable', + vary: 'Accept', + link: '; rel="cite-as"', + }, + }); + assert.equal(decide('/release/v2026.10.01', null).originPath, '/release/v2026.10.01/index.jsonld'); + assert.deepEqual(decide('/release/v2026.10.01', 'text/html,application/xhtml+xml,*/*;q=0.8'), { + originPath: '/release/v2026.10.01/index.html', + headers: { + 'content-type': 'text/html; charset=utf-8', + 'cache-control': 'public, max-age=300', + vary: 'Accept', + link: '; rel="canonical", ; rel="cite-as"', + }, + }); +}); + +test('decide: /release lists releases, negotiated, and gains members so it keeps the short cache', () => { + assert.deepEqual(decide('/release', '*/*'), { originPath: '/release/index.jsonld', headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.deepEqual(decide('/release', 'text/html'), { originPath: '/release/index.html', headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); +}); + +test('decide: anything that is not exactly a release tag falls through', () => { + for (const p of ['/release/', '/release/latest', '/release/2026.10.01', '/release/v2026.10', '/release/v2026.10.1', '/release/v2026.10.01/', + '/release/v2026.10.01/index.jsonld', '/release/v2026.10.01.tar.gz', '/release/draft-base', '/releases', '/release/v2026.10.01/x']) { + assert.deepEqual(decide(p, 'text/html'), { originPath: p, headers: { 'cache-control': 'public, max-age=300' } }, `${p} must pass through`); + } +}); + +test('worker fetch handler: an unknown release tag 404s with the short negative cache, a known one is typed', async () => { + const seen: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input); + seen.push(url); + return url.endsWith('/release/v2026.10.01/index.jsonld') + ? new Response('{"@id":"x"}', { status: 200, headers: { 'content-type': 'application/octet-stream' } }) + : new Response('nope', { status: 404 }); + }) as typeof fetch; + try { + const env = { ORIGIN: 'https://origin.example/dictionary' }; + const known = await worker.fetch(new Request('https://material-identity.eu/release/v2026.10.01'), env); + assert.equal(known.status, 200); + assert.equal(known.headers.get('content-type'), 'application/ld+json'); + assert.equal(known.headers.get('cache-control'), 'public, max-age=31536000, immutable'); + const unknown = await worker.fetch(new Request('https://material-identity.eu/release/v2099.01.01'), env); + assert.equal(unknown.status, 404); + assert.equal(unknown.headers.get('cache-control'), 'public, max-age=60', 'a 404 is never cached as immutable'); + assert.deepEqual(seen, ['https://origin.example/dictionary/release/v2026.10.01/index.jsonld', 'https://origin.example/dictionary/release/v2099.01.01/index.jsonld']); + } finally { + globalThis.fetch = realFetch; + } +}); diff --git a/test/release.test.ts b/test/release.test.ts new file mode 100644 index 0000000..bbe28a5 --- /dev/null +++ b/test/release.test.ts @@ -0,0 +1,210 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cpSync, existsSync, mkdtempSync, readFileSync, renameSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import oxigraph from 'oxigraph'; +import { build } from '../scripts/build.ts'; +import { buildCatalog, buildRelease, buildReleaseIndex, releaseContents } from '../scripts/lib/dcat.ts'; +import { entrySha256 } from '../scripts/lib/emit.ts'; +import { getReleases, getReleaseTags, gitEnv, type Release, shippedBy } from '../scripts/lib/git.ts'; +import { loadRepo } from '../scripts/lib/repo.ts'; +import { pocBanner, RefIndex, renderReleaseIndexPage, renderReleasePage } from '../scripts/lib/render.ts'; +import { distributionPins, loadSpecs, parseJsonLd } from '../scripts/lib/spec.ts'; +import { runValidation } from '../scripts/validate.ts'; + +// Dictionary releases at /release/ (#135): each release tag as a dcat:Dataset. +const here = dirname(fileURLToPath(import.meta.url)); +const GREEN = join(here, 'fixtures', 'green'); +const BASE = 'https://material-identity.eu'; +const DCAT = 'http://www.w3.org/ns/dcat#'; +const DCT = 'http://purl.org/dc/terms/'; +const SPDX = 'http://spdx.org/rdf/terms#'; +const LATE = 'published/c38a85eb-1a37-416d-ab21-7ddcc599754d.yaml'; + +const ask = (store: oxigraph.Store, where: string): boolean => store.query(`ASK { ${where} }`) as unknown as boolean; +const count = (store: oxigraph.Store, where: string): number => + Number((store.query(`SELECT (COUNT(*) AS ?n) WHERE { ${where} }`) as Array>)[0].get('n')!.value); +const sha = (bytes: Buffer | string): string => createHash('sha256').update(bytes).digest('hex'); +const tag = (t: string): Release => ({ tag: t, date: t.slice(1).replaceAll('.', '-') }); + +/** The green repo, with every entry first shipped in v2026.08.28 except LATE, which waits for v2026.09.30. */ +function greenReleases(): { repo: ReturnType; tags: Release[]; first: Map } { + const repo = loadRepo(GREEN); + const first = new Map(repo.published.map((f) => [f.relPath, tag(f.relPath === LATE ? 'v2026.09.30' : 'v2026.08.28')])); + return { repo, tags: [tag('v2026.09.30'), tag('v2026.08.28')], first }; +} + +test('shippedBy: a release contains what it or an earlier release first shipped, nothing unreleased', () => { + assert.equal(shippedBy(tag('v2026.08.28'), 'v2026.08.28'), true); + assert.equal(shippedBy(tag('v2026.08.28'), 'v2026.09.30'), true); + assert.equal(shippedBy(tag('v2026.09.30'), 'v2026.08.28'), false); + assert.equal(shippedBy(undefined, 'v2099.01.01'), false); +}); + +test('releaseContents: cumulative per tag, sorted, with the digest a spec pins; unreleased entries in none', () => { + const { repo, tags, first } = greenReleases(); + first.delete('published/109980b4-51dc-4986-ba73-d5e407318400.yaml'); // merged since the last tag + const [early, late] = releaseContents(tags, first, repo); + assert.deepEqual([early.tag, early.date, late.tag], ['v2026.08.28', '2026-08-28', 'v2026.09.30']); + const released = repo.published.length - 1; + assert.equal(late.entries.length, released); + assert.equal(early.entries.length, released - 1); + assert.ok(!early.entries.some((e) => e.iri.endsWith('c38a85eb-1a37-416d-ab21-7ddcc599754d'))); + assert.ok(![...early.entries, ...late.entries].some((e) => e.iri.endsWith('109980b4-51dc-4986-ba73-d5e407318400'))); + assert.deepEqual(late.entries.map((e) => e.iri), [...late.entries.map((e) => e.iri)].sort()); + assert.deepEqual(late.entries.filter((e) => e.firstShipped === 'v2026.09.30').map((e) => e.iri), [`${BASE}/def/c38a85eb-1a37-416d-ab21-7ddcc599754d`]); + for (const e of late.entries) { + const doc = repo.published.find((f) => e.iri.endsWith(f.stem))!.doc!; + assert.equal(e.sha256, entrySha256(doc)); + } +}); + +test('a release is a PoC-marked, CC0 dcat:Dataset: entries it contains, entries it first shipped, the tarball by link', () => { + const { repo, tags, first } = greenReleases(); + const [, late] = releaseContents(tags, first, repo); + const iri = `${BASE}/release/v2026.09.30`; + const store = parseJsonLd(JSON.stringify(buildRelease(late)), iri); + assert.ok(ask(store, `<${iri}> a <${DCAT}Dataset> ; <${DCAT}version> "v2026.09.30" ; + <${DCT}issued> "2026-09-30"^^ ; + <${DCT}license> ; + <${BASE}/ns#ProofOfConcept> ; + <${DCAT}inCatalog> <${BASE}/release> ; + <${DCAT}landingPage> .`)); + assert.ok(ask(store, `<${iri}> <${DCAT}distribution> ?d . ?d a <${DCAT}Distribution> ; + <${DCAT}downloadURL> ; + <${DCAT}mediaType> "application/gzip" . FILTER NOT EXISTS { ?d <${SPDX}checksum> ?c }`), 'the tarball is linked, with no digest the offline build cannot know'); + assert.equal(count(store, `<${iri}> <${DCT}hasPart> ?e`), late.entries.length); + assert.equal(count(store, `<${iri}> <${BASE}/ns#firstShipped> ?e`), 1); + for (const e of late.entries) { + assert.ok(ask(store, `<${iri}> <${DCT}hasPart> <${e.iri}> . <${e.iri}> <${DCAT}distribution> ?d . + ?d <${DCAT}downloadURL> <${e.iri}> ; <${DCAT}mediaType> "application/json" ; + <${SPDX}checksum> [ <${SPDX}algorithm> <${SPDX}checksumAlgorithm_sha256> ; <${SPDX}checksumValue> "${e.sha256}" ] .`), e.iri); + } + // deterministic, whatever order the inputs come in + assert.equal(JSON.stringify(releaseContents([...tags].reverse(), first, repo)), JSON.stringify(releaseContents(tags, first, repo))); +}); + +test('/release and /catalog describe each release with the same node and IRI as /release/', () => { + const { repo, tags, first } = greenReleases(); + const index = buildReleaseIndex(tags); + assert.equal(index['@id'], `${BASE}/release`); + assert.equal(index['@type'], 'dcat:Catalog'); + const listed = index['dcat:dataset'] as Array>; + assert.deepEqual(listed.map((d) => d['@id']), [`${BASE}/release/v2026.08.28`, `${BASE}/release/v2026.09.30`]); + const inCatalog = (buildCatalog(tags, [])['dcat:dataset'] as Array>); + assert.deepEqual(inCatalog, listed); + for (const [i, r] of releaseContents(tags, first, repo).entries()) { + const own = buildRelease(r); + for (const [k, v] of Object.entries(listed[i])) assert.deepEqual(own[k], v, `${r.tag} ${k}`); + } + const store = parseJsonLd(JSON.stringify(index), `${BASE}/release`); + assert.ok(ask(store, `<${BASE}/release> <${BASE}/ns#ProofOfConcept> ; <${DCAT}dataset> <${BASE}/release/v2026.08.28> .`)); +}); + +test('release pages carry the PoC banner, name themselves canonical, and link every entry with its digest', () => { + const { repo, tags, first } = greenReleases(); + const refs = new RefIndex(repo); + const contents = releaseContents(tags, first, repo); + const html = renderReleasePage(contents[1], refs); + assert.ok(html.includes(pocBanner())); + assert.match(html, //); + assert.match(html, //); + assert.match(html, /

Release v2026\.09\.30<\/h1>/); + assert.match(html, /href="https:\/\/github\.com\/material-identity\/dictionary\/releases\/download\/v2026\.09\.30\/dictionary-v2026\.09\.30\.tar\.gz"/); + assert.match(html, new RegExp(`${contents[1].entries.length} in total, 1 first shipped in this release`)); + assert.match(html, /v2026\.08\.28<\/code><\/a><\/td>/, 'an older entry links the release that first shipped it'); + for (const e of contents[1].entries) assert.ok(html.includes(`${e.sha256}`)); + assert.ok(!/