From 2ecb20a9386f0a48f65edad395cc7a2b801d9dc4 Mon Sep 17 00:00:00 2001 From: Hannes Stiebitzhofer Date: Mon, 2 Nov 2026 22:59:12 +0100 Subject: [PATCH 1/6] Host DPP content specifications, the DCAT catalogue and the mi: vocabulary (#135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit material-identity/schema generates each content specification; this repo hosts it byte-identical (design: schema docs/design/content-spec-jsonld.md §2.1, §2.3, §3.3, §7). Everything about it is a proof of concept and says so in-band and on every page. - spec/// is a new add-only source tree holding the hand-over as is: .jsonld, .context.jsonld, .dcat.jsonld and the two derived schemas. lib/spec.ts loads it through oxigraph with no document loader, so a remote @context fails: the spec's own context must be inline. - Worker: /spec// (JSON-LD default, HTML; immutable; Link cite-as), its context.jsonld and compressed/full schemas (immutable), /spec/ and /catalog (JSON-LD or HTML, short cache), /ns (JSON-LD, Turtle, HTML, short cache). Strict act and version shapes; anything else falls through. Negotiated routes send Vary: Accept. - Build: spec files byte-identical at the origin paths the Worker maps onto, an HTML page per spec (bilingual PoC banner, metadata, SHA-256 of every file, pins, memberships in sh:order), the /spec/ series, /catalog and /ns. - lib/dcat.ts: a dcat:Catalog of the dictionary releases (vYYYY.MM.DD tags), the act series and each spec's own DCAT record, embedded with the record's @context so its checksums keep the meaning they were generated with. Deterministic. - Checks: check 1 covers spec/ (add-only). A new unnumbered spec check refuses bad layout, unparseable JSON-LD, a spec not mi:ProofOfConcept or not CC0, an @id/version/act that disagrees with its directory, a pin outside published/ or whose digest differs from the served /def/.json, and DCAT checksums that differ from the hosted bytes. - Vocabulary: the design's new mi: terms join LOCAL_TERMS with label and comment; individuals (granularity levels, ProofOfConcept) are skos:Concepts. Terms are now rdfs:isDefinedBy <…/ns>, the resolvable document, and properties are typed rdf:Property (was the non-existent rdfs:Property). Tested against a small fixture spec under test/fixtures/green/spec/, never under the repo's own spec/, which only ever holds hand-overs. Co-Authored-By: Claude Opus 5.5 --- scripts/build.ts | 44 +- scripts/lib/checks.ts | 79 +++- scripts/lib/dcat.ts | 152 +++++++ scripts/lib/git.ts | 22 + scripts/lib/rdf.ts | 95 ++++- scripts/lib/render.ts | 200 ++++++++- scripts/lib/spec.ts | 312 ++++++++++++++ scripts/lib/styles.css | 6 + test/checks.test.ts | 2 +- .../spec/demo/v0.0.1/compressed.schema.json | 11 + .../spec/demo/v0.0.1/demo.context.jsonld | 16 + .../green/spec/demo/v0.0.1/demo.dcat.jsonld | 56 +++ .../green/spec/demo/v0.0.1/demo.jsonld | 90 ++++ .../green/spec/demo/v0.0.1/full.schema.json | 8 + test/index-worker.test.ts | 81 ++++ test/spec.test.ts | 402 ++++++++++++++++++ test/validate.test.ts | 4 +- worker/index.ts | 63 ++- 18 files changed, 1620 insertions(+), 23 deletions(-) create mode 100644 scripts/lib/dcat.ts create mode 100644 scripts/lib/spec.ts create mode 100644 test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json create mode 100644 test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld create mode 100644 test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld create mode 100644 test/fixtures/green/spec/demo/v0.0.1/demo.jsonld create mode 100644 test/fixtures/green/spec/demo/v0.0.1/full.schema.json create mode 100644 test/spec.test.ts diff --git a/scripts/build.ts b/scripts/build.ts index 9c0ef33..cbabdb0 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -8,11 +8,13 @@ import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { loadRepo } from './lib/repo.ts'; import { canonicalJson } from './lib/emit.ts'; -import { type ExampleSpec, RefIndex, renderAboutPage, renderEntryPage, renderExamplesPage, renderGraphPage, renderIndexPages, renderSchemaPage, renderTreePage } from './lib/render.ts'; +import { type ExampleSpec, RefIndex, renderAboutPage, renderCatalogPage, renderEntryPage, renderExamplesPage, renderGraphPage, renderIndexPages, renderSchemaPage, renderSeriesPage, renderSpecPage, renderTreePage, renderVocabularyPage } from './lib/render.ts'; import { renderFeed } from './lib/feed.ts'; import { citation } from './lib/cite.ts'; -import { renderTurtle } from './lib/rdf.ts'; -import { getAddedDates, getReleases } from './lib/git.ts'; +import { renderTurtle, renderVocabularyJsonLd, renderVocabularyTurtle, vocabularyTerms } from './lib/rdf.ts'; +import { getAddedDates, getReleases, getReleaseTags } from './lib/git.ts'; +import { buildCatalog, buildSeries, byAct, jsonLd } from './lib/dcat.ts'; +import { type HostedSpec, loadSpecs, summarize } from './lib/spec.ts'; const LIB_DIR = dirname(fileURLToPath(import.meta.url)); // The schema is one canonical file for the whole site, independent of which content tree @@ -89,9 +91,45 @@ export function build(root: string, out: string): BuildResult { } const issued = new Map([...releases].map(([path, release]) => [path, release.date])); writeFileSync(join(out, 'dictionary.ttl'), renderTurtle(repo, refs, issued)); + + // The mi: vocabulary, resolvable at /ns (#135): the same term list /dictionary.ttl describes. + mkdirSync(join(out, 'ns'), { recursive: true }); + writeFileSync(join(out, 'ns', 'index.ttl'), renderVocabularyTurtle()); + writeFileSync(join(out, 'ns', 'index.jsonld'), renderVocabularyJsonLd()); + writeFileSync(join(out, 'ns', 'index.html'), renderVocabularyPage(vocabularyTerms())); + + const specs = emitSpecs(root, out, refs); + const tags = getReleaseTags(root); + mkdirSync(join(out, 'catalog'), { recursive: true }); + writeFileSync(join(out, 'catalog', 'index.jsonld'), jsonLd(buildCatalog(tags, specs))); + writeFileSync(join(out, 'catalog', 'index.html'), renderCatalogPage(tags, specs.map((spec) => ({ spec, summary: summarize(spec) })))); return { entries, out }; } +/** + * Hosted content specifications (#135): `spec///` → `site/spec///`, + * every served file byte-identical under the name the Worker maps its IRI onto, plus an HTML page + * per spec and a JSON-LD + HTML series index per act. Refuses a spec tree that does not load. + */ +export function emitSpecs(root: string, out: string, refs: RefIndex): HostedSpec[] { + const { specs, issues } = loadSpecs(root); + if (issues.length > 0) { + throw new Error(`refusing to build from an unloadable spec/ tree:\n${issues.map((i) => `${i.file}: ${i.message}`).join('\n')}`); + } + for (const spec of specs) { + const dir = join(out, spec.relDir); + mkdirSync(dir, { recursive: true }); + for (const file of spec.files) if (file.served !== undefined) writeFileSync(join(dir, file.served), file.bytes); + writeFileSync(join(dir, 'index.html'), renderSpecPage(spec, summarize(spec), refs)); + } + for (const [act, versions] of byAct(specs)) { + const dir = join(out, 'spec', act); + writeFileSync(join(dir, 'index.jsonld'), jsonLd(buildSeries(act, versions))); + writeFileSync(join(dir, 'index.html'), renderSeriesPage(act, versions.map((spec) => ({ spec, summary: summarize(spec) })))); + } + return specs; +} + /** Every examples/content-specifications/*.json, sorted, with the SHA-256 of its exact bytes. */ export function loadExampleSpecs(dir: string): ExampleSpec[] { const specDir = join(dir, 'content-specifications'); diff --git a/scripts/lib/checks.ts b/scripts/lib/checks.ts index 67db4d0..bdaaec5 100644 --- a/scripts/lib/checks.ts +++ b/scripts/lib/checks.ts @@ -1,4 +1,5 @@ import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,6 +9,10 @@ import addFormatsModule from 'ajv-formats'; import { validate as uuidValidate, version as uuidVersion } from 'uuid'; import { DEF_PREFIX, type RepoFile, type RepoModel, type ValidationIssue } from './repo.ts'; import type { DiffEntry, GitContext } from './git.ts'; +import { canonicalJson } from './emit.ts'; +import { embeddedDataset } from './dcat.ts'; +import { CC0 } from './rdf.ts'; +import { ask, distributionPins, loadSpecs, type SpecLoad } from './spec.ts'; // ajv/ajv-formats ship CJS; under native ESM the class may sit on .default or be the export itself. const Ajv2019 = (Ajv2019Module as unknown as { default?: typeof Ajv2019Module }).default ?? Ajv2019Module; @@ -286,17 +291,23 @@ export function checkPgpKey(root: string, now: Date = new Date(), horizonDays = return pgpKeyExpiryIssues(colons, now, horizonDays).map((message) => ({ check, file, message })); } -/** Check 1 — immutability (R6): only additions are allowed under published/. */ +/** + * Check 1 — immutability (R6): only additions are allowed under published/, and under spec/ — a + * hosted content specification is served immutable, so a change is a new contract version (#135). + */ export function checkImmutability(diff: DiffEntry[]): ValidationIssue[] { const issues: ValidationIssue[] = []; for (const entry of diff) { - if (!entry.path.startsWith('published/')) continue; + const spec = entry.path.startsWith('spec/'); + if (!spec && !entry.path.startsWith('published/')) continue; if (entry.status === 'A') continue; const verb = entry.status === 'D' ? 'deleted' : entry.status === 'M' ? 'modified' : `changed (${entry.status})`; issues.push({ check: '1 immutability', file: entry.path, - message: `published file ${verb} — published/ is add-only (R6); publish a new entry with replaces instead`, + message: spec + ? `hosted specification file ${verb} — spec/ is add-only (#135); a change is a new contract version under a new directory` + : `published file ${verb} — published/ is add-only (R6); publish a new entry with replaces instead`, }); } return issues; @@ -355,6 +366,64 @@ export function checkMovePurity(repo: RepoModel, git: GitContext): ValidationIss return issues; } +/** + * Hosted content specifications (#135). This repo hosts what material-identity/schema generated + * and does not re-validate content (design §7.2 step 4) — but it refuses to serve, as immutable, + * a file that is not what it claims to be: JSON-LD that does not parse with an inline context, a + * spec that is not marked as a proof of concept or not CC0, an id that disagrees with where it is + * served, a pin to an entry this dictionary has not published (or whose served bytes differ from + * the pinned digest), or a DCAT record whose checksums do not match the bytes served here. + */ +export function checkSpecs(load: SpecLoad, repo: RepoModel): ValidationIssue[] { + const issues: ValidationIssue[] = [...load.issues]; + const check = 'spec'; + const published = new Map(repo.published.filter((f) => f.doc).map((f) => [f.stem, f.doc!])); + + for (const spec of load.specs) { + const file = `${spec.relDir}/${spec.act}.jsonld`; + const add = (message: string, at = file): void => { issues.push({ check, file: at, message }); }; + const g = spec.graph; + const s = `<${spec.iri}>`; + + if (!ask(g, `${s} a mi:ContentSpecification`)) add(`no node ${spec.iri} typed mi:ContentSpecification — the @id must be the IRI the spec is served at`); + if (!ask(g, `${s} adms:status mi:ProofOfConcept`)) add('adms:status is not mi:ProofOfConcept — every hosted specification is a proof of concept and must say so in-band'); + if (!ask(g, `${s} dcterms:license <${CC0}>`)) add(`dcterms:license is not CC0 (<${CC0}>)`); + if (!ask(g, `${s} dcat:version "${spec.version}"`)) add(`dcat:version is not "${spec.version}", the version directory it is served from`); + if (!ask(g, `${s} mi:act "${spec.act}"`)) add(`mi:act is not "${spec.act}", the act directory it is served from`); + + const pins = distributionPins(g, spec.iri); + if (pins.length === 0) add('pins no dictionary entry (dcterms:requires → dcat:Distribution with a dcat:downloadURL)'); + for (const pin of pins) { + const uuid = defUuid(pin.url); + const doc = uuid === undefined ? undefined : published.get(uuid); + if (doc === undefined) { + add(`pinned ${pin.url} is not an entry in published/`); + } else if (pin.sha256 === undefined) { + add(`pinned ${pin.url} carries no spdx:checksum — a pin without a digest is never publishable`); + } else { + const served = createHash('sha256').update(canonicalJson(doc)).digest('hex'); + if (pin.sha256 !== served) add(`pinned ${pin.url} has checksum ${pin.sha256}, but /def/${uuid} serves ${served}`); + } + } + + const dcatFile = `${spec.relDir}/${spec.act}.dcat.jsonld`; + const d = spec.dcatGraph; + if (embeddedDataset(spec) === undefined) add(`the DCAT record has no node for ${spec.iri}`, dcatFile); + if (!ask(d, `${s} a dcat:Dataset`)) add(`the DCAT record does not describe ${spec.iri} as a dcat:Dataset`, dcatFile); + if (!ask(d, `${s} adms:status mi:ProofOfConcept`)) add('the DCAT record does not mark the spec mi:ProofOfConcept', dcatFile); + if (!ask(d, `${s} dcterms:license <${CC0}>`)) add(`the DCAT record's dcterms:license is not CC0 (<${CC0}>)`, dcatFile); + const served = new Map(spec.files.filter((f) => f.served !== undefined).map((f) => [f.url, f])); + const distributions = distributionPins(d, spec.iri, 'dcat:distribution'); + if (!distributions.some((dist) => dist.url === spec.iri)) add(`the DCAT record lists no distribution of the spec itself (dcat:downloadURL <${spec.iri}>)`, dcatFile); + for (const dist of distributions) { + const hosted = served.get(dist.url); + if (hosted === undefined) add(`distribution ${dist.url} is not a file hosted for this spec`, dcatFile); + else if (dist.sha256 !== hosted.sha256) add(`distribution ${dist.url} has checksum ${dist.sha256 ?? '(none)'}, but the hosted ${hosted.source} hashes to ${hosted.sha256}`, dcatFile); + } + } + return issues; +} + export interface CheckResult { name: string; issues: ValidationIssue[]; @@ -363,7 +432,7 @@ export interface CheckResult { } /** - * Run all validate.ts checks (1–6, plus the unnumbered security.txt guard). Checks 1 and 6 need + * Run all validate.ts checks (1–6, plus the unnumbered hosted-spec and security.txt guards). Checks 1 and 6 need * a git context. Check 7 (two-yes gate) lives in CI only. */ export function runChecks(repo: RepoModel, git?: GitContext, horizonDays = 0): CheckResult[] { @@ -377,6 +446,8 @@ export function runChecks(repo: RepoModel, git?: GitContext, horizonDays = 0): C { name: 'check 3 — identity', issues: checkIdentity(repo) }, { name: 'check 4 — replaces', issues: checkReplaces(repo) }, { name: 'check 5 — pinning', issues: checkPinning(repo) }, + // Unnumbered like security.txt: 1–6 are the entry rules. Hosted specs ride on check 1 (add-only). + { name: 'spec — hosted content specifications', issues: checkSpecs(loadSpecs(repo.root), repo) }, git ? { name: 'check 6 — move purity', issues: checkMovePurity(repo, git) } : { name: 'check 6 — move purity', issues: [], skipped: noGit }, diff --git a/scripts/lib/dcat.ts b/scripts/lib/dcat.ts new file mode 100644 index 0000000..ab6897b --- /dev/null +++ b/scripts/lib/dcat.ts @@ -0,0 +1,152 @@ +/** + * The DCAT catalogue at /catalog and the version series at /spec/ (#135; schema design + * content-spec-jsonld.md §2.1, §7.3). Both are derived at build time, like /feed.xml: a new + * dictionary release or a newly hosted specification joins them, so they get a short cache. + * + * A hosted spec enters the catalogue through its own `.dcat.jsonld` record, merged as + * written: the record's dataset node is embedded with the record's own `@context` (JSON-LD 1.1 + * allows a context on any node object), so its terms and its checksums keep exactly the meaning + * the schema repo generated. Nothing here recomputes or rewrites a digest. + * + * Deterministic: releases sorted by tag, specs by act then version, keys in a fixed order. + * Everything is a proof of concept and says so in-band (`adms:status mi:ProofOfConcept`). + */ +import { CANONICAL_BASE } from './repo.ts'; +import { ADMS, CC0, VOCABULARY } from './rdf.ts'; +import { type HostedSpec, summarize } from './spec.ts'; + +export const CATALOG_IRI = `${CANONICAL_BASE}/catalog`; +export const releaseIri = (tag: string): string => `${CANONICAL_BASE}/release/${tag}`; +export const seriesIri = (act: string): string => `${CANONICAL_BASE}/spec/${act}`; +const releaseLandingPage = (tag: string): string => `https://github.com/material-identity/dictionary/releases/tag/${tag}`; + +export const CONTEXT = { + '@version': 1.1, + dcat: 'http://www.w3.org/ns/dcat#', + dcterms: 'http://purl.org/dc/terms/', + adms: ADMS, + spdx: 'http://spdx.org/rdf/terms#', + xsd: 'http://www.w3.org/2001/XMLSchema#', + mi: `${VOCABULARY}#`, +}; + +export const POC_DESCRIPTION = { + en: 'PROOF OF CONCEPT. Nothing listed here is a content specification of the European Union, adopted under Regulation (EU) 2024/1781, or otherwise normative.', + de: 'PROOF OF CONCEPT. Nichts hier Aufgeführte ist eine Inhaltsspezifikation der Europäischen Union, nach Verordnung (EU) 2024/1781 erlassen oder anderweitig normativ.', +}; + +const langLiterals = (map: Record): Array<{ '@value': string; '@language': string }> => + Object.entries(map) + .filter(([code]) => code !== '') + .sort(([a], [b]) => (a === 'en' ? -1 : b === 'en' ? 1 : a.localeCompare(b, 'en'))) + .map(([code, value]) => ({ '@value': value, '@language': code })); + +const POC = { 'adms:status': { '@id': 'mi:ProofOfConcept' }, 'dcterms:license': { '@id': CC0 } }; + +export interface CatalogRelease { + tag: string; + /** YYYY-MM-DD */ + date: string; +} + +/** + * The record's node for `iri`, wherever it sits (the record root itself, or one `dcat:dataset` of + * a catalogue-shaped record as in design §7.3). The first node with that `@id` and more than an id. + */ +export function findDatasetNode(record: unknown, iri: string): Record | undefined { + if (Array.isArray(record)) { + for (const item of record) { + const hit = findDatasetNode(item, iri); + if (hit) return hit; + } + return undefined; + } + if (!record || typeof record !== 'object') return undefined; + const node = record as Record; + if (node['@id'] === iri && Object.keys(node).some((k) => k !== '@id' && k !== '@context')) return node; + for (const [k, v] of Object.entries(node)) { + if (k === '@context') continue; + const hit = findDatasetNode(v, iri); + if (hit) return hit; + } + return undefined; +} + +/** A spec's dataset node from its DCAT record, carrying the record's context so it reads the same. */ +export function embeddedDataset(spec: HostedSpec): Record | undefined { + const node = findDatasetNode(spec.dcatJson, spec.iri); + if (!node) return undefined; + const outer = spec.dcatJson['@context']; + const inner = node === spec.dcatJson ? undefined : node['@context']; + const context = inner === undefined ? outer : [outer, inner].flat(); + const { '@context': _ignored, ...rest } = node; + return context === undefined ? rest : { '@context': context, ...rest }; +} + +/** Group specs by act, acts sorted, versions in the order loadSpecs gives them (numeric). */ +export function byAct(specs: HostedSpec[]): Map { + const acts = new Map(); + for (const spec of [...specs].sort((a, b) => a.act.localeCompare(b.act, 'en'))) { + acts.set(spec.act, [...(acts.get(spec.act) ?? []), spec]); + } + return acts; +} + +/** /spec/: every hosted version of one act, as a DCAT dataset series. */ +export function buildSeries(act: string, versions: HostedSpec[]): Record { + return { + '@context': CONTEXT, + '@id': seriesIri(act), + '@type': 'dcat:DatasetSeries', + 'dcterms:title': { '@value': `${act} — DPP content specifications (PoC)`, '@language': 'en' }, + 'dcterms:description': langLiterals(POC_DESCRIPTION), + ...POC, + 'mi:act': act, + 'dcat:seriesMember': versions.map((spec) => { + const s = summarize(spec); + const member: Record = { '@id': spec.iri, 'dcat:version': spec.version }; + if (s.issued !== undefined) member['dcterms:issued'] = { '@value': s.issued, '@type': 'xsd:date' }; + const title = langLiterals(s.title); + if (title.length > 0) member['dcterms:title'] = title; + return member; + }), + // DCAT 3 versioning: the inverse of each spec's own dcat:isVersionOf. + 'dcat:hasVersion': versions.map((spec) => ({ '@id': spec.iri })), + }; +} + +/** /catalog: the dictionary releases, the spec series, and every hosted spec's own DCAT record. */ +export function buildCatalog(releases: CatalogRelease[], specs: HostedSpec[]): Record { + const releaseDatasets = [...releases].sort((a, b) => a.tag.localeCompare(b.tag, 'en')).map((r) => ({ + '@id': releaseIri(r.tag), + '@type': 'dcat:Dataset', + 'dcterms:title': { '@value': `material-identity dictionary, release ${r.tag}`, '@language': 'en' }, + 'dcat:version': r.tag, + 'dcterms:issued': { '@value': r.date, '@type': 'xsd:date' }, + ...POC, + 'dcat:landingPage': { '@id': releaseLandingPage(r.tag) }, + })); + const acts = byAct(specs); + const series = [...acts.keys()].map((act) => ({ + '@id': seriesIri(act), + '@type': 'dcat:DatasetSeries', + 'dcterms:title': { '@value': `${act} — DPP content specifications (PoC)`, '@language': 'en' }, + ...POC, + })); + const specDatasets = [...acts.values()].flat().map((spec) => embeddedDataset(spec) ?? { '@id': spec.iri }); + return { + '@context': CONTEXT, + '@id': CATALOG_IRI, + '@type': 'dcat:Catalog', + 'dcterms:title': [ + { '@value': 'material-identity — dictionary releases and DPP content specifications (PoC)', '@language': 'en' }, + { '@value': 'material-identity — Wörterbuch-Releases und DPP-Inhaltsspezifikationen (PoC)', '@language': 'de' }, + ], + 'dcterms:description': langLiterals(POC_DESCRIPTION), + ...POC, + 'dcterms:publisher': { '@id': `${CANONICAL_BASE}/` }, + 'dcat:dataset': [...releaseDatasets, ...series, ...specDatasets], + }; +} + +export const jsonLd = (doc: Record): string => `${JSON.stringify(doc, null, 2)}\n`; diff --git a/scripts/lib/git.ts b/scripts/lib/git.ts index 641b92a..72872c2 100644 --- a/scripts/lib/git.ts +++ b/scripts/lib/git.ts @@ -136,6 +136,28 @@ export interface Release { date: string; } +/** A dictionary release tag: `vYYYY.MM.DD` (other tags, e.g. `draft-base`, are not releases). */ +export const RELEASE_TAG = /^v\d{4}\.\d{2}\.\d{2}$/; + +/** + * Every dictionary release, sorted by tag, for the DCAT catalogue (#135). Silently empty outside a + * git work-tree top level, like getAddedCommits — a fixture-tree build lists no release. + */ +export function getReleaseTags(root: string): Release[] { + if (typeof requireToplevel(root) !== 'string') return []; + let output: string; + try { + output = git(root, ['tag', '--list', '--format=%(refname:short) %(creatordate:short)']); + } catch { + return []; + } + return output.split('\n') + .map((line) => line.trim().split(' ')) + .filter(([tag, date]) => RELEASE_TAG.test(tag ?? '') && date !== undefined) + .map(([tag, date]) => ({ tag, date })) + .sort((a, b) => a.tag.localeCompare(b.tag, 'en')); +} + /** * The first tagged release containing each published file (issue #94): an entry is cited by the * release it shipped in, not by the day its commit happened to land. Entries merged since the diff --git a/scripts/lib/rdf.ts b/scripts/lib/rdf.ts index 5590e08..3050962 100644 --- a/scripts/lib/rdf.ts +++ b/scripts/lib/rdf.ts @@ -15,6 +15,10 @@ import { DEF_PREFIX, type RepoModel } from './repo.ts'; import { lang, type Doc, type RefIndex } from './render.ts'; const NS = 'https://material-identity.eu/ns#'; +/** The vocabulary document, served at /ns (#135); its terms are `#`. */ +export const VOCABULARY = 'https://material-identity.eu/ns'; +export const CC0 = 'http://creativecommons.org/publicdomain/zero/1.0/'; +export const ADMS = 'http://www.w3.org/ns/adms#'; const PREFIXES: Array<[string, string]> = [ ['mi', NS], @@ -27,7 +31,8 @@ const PREFIXES: Array<[string, string]> = [ ]; /** The local terms, described in the graph itself so the file needs no companion document. */ -const LOCAL_TERMS: Array<[term: string, kind: 'Class' | 'Property', label: string, comment: string]> = [ +export type TermKind = 'Class' | 'Property' | 'Individual'; +const LOCAL_TERMS: Array<[term: string, kind: TermKind, label: string, comment: string]> = [ ['SingleValuedDataElement', 'Class', 'Single-valued data element', 'Entry kind: one value of the given data type.'], ['MultiLanguageDataElement', 'Class', 'Multi-language data element', 'Entry kind: one value per language.'], ['MultiValuedDataElement', 'Class', 'Multi-valued data element', 'Entry kind: a list whose items all have the item type.'], @@ -57,6 +62,24 @@ const LOCAL_TERMS: Array<[term: string, kind: 'Class' | 'Property', label: strin ['toUnit', 'Property', 'to unit', 'Target unit of a conversion.'], ['factor', 'Property', 'factor', 'Multiplicative factor of a conversion.'], ['offset', 'Property', 'offset', 'Additive offset of a conversion.'], + // Content-specification terms (#135; schema design content-spec-jsonld.md §3.3). A spec is a + // SHACL shapes graph; these say what SHACL, DCAT and DCTerms cannot. + ['ContentSpecification', 'Class', 'DPP content specification', 'The set of data elements a passport for one product group carries, published as one SHACL shapes graph. EN 18223 defines no such class.'], + ['elementId', 'Property', 'element id', 'The EN 18223 element key of a membership: the JSON key in a passport and the Annex A elementId. A key, not a display label.'], + ['granularity', 'Property', 'granularity', 'The level a value is given at: mi:model, mi:batch or mi:item (EN 18219), or mi:operator for economic-operator data. Used on memberships and on the passport header.'], + ['model', 'Individual', 'model', 'Granularity: one value for every item of a product model.'], + ['batch', 'Individual', 'batch', 'Granularity: one value per batch or heat.'], + ['item', 'Individual', 'item', 'Granularity: one value per individual item.'], + ['operator', 'Individual', 'operator', 'Granularity: data about the economic operator, not the product. Never taken by a passport header.'], + ['condition', 'Property', 'condition', 'The condition under which an optional membership becomes mandatory, per language. Stated, not evaluated: SHACL has no way to express it.'], + ['ProofOfConcept', 'Individual', 'proof of concept', 'Value of adms:status: a demonstration resource, never normative. A content specification with this status is not one of the European Union.'], + ['act', 'Property', 'act', 'The delegated-act key of a content specification, as used in its IRI and file names.'], + ['dppSchemaVersion', 'Property', 'DPP schema version', 'The EN 18223 Table 1 field naming the metamodel envelope a passport or content specification is derived against.'], + ['passportContext', 'Property', 'passport context', 'Links a content specification to the JSON-LD context its passports are read with.'], + ['uniqueProductIdentifier', 'Property', 'unique product identifier', 'Passport header: the unique product identifier (EN 18223 Table 1).'], + ['dppStatus', 'Property', 'DPP status', 'Passport header: the status of the passport (EN 18223 Table 1).'], + ['economicOperatorId', 'Property', 'economic operator id', 'Passport header: the identifier of the responsible economic operator (EN 18223 Table 1).'], + ['facilityId', 'Property', 'facility id', 'Passport header: the identifier of the facility (EN 18223 Table 1).'], ]; function literal(value: string): string { @@ -180,13 +203,20 @@ function entryTriples(uuid: string, doc: Doc, refs: RefIndex, issued: string | u return `${iri(doc.id)}\n${block(pairs)} .`; } +/** What `a` a local term is: individuals (granularity levels, a status value) are SKOS concepts. */ +function termType(kind: TermKind): string { + return kind === 'Class' ? 'rdfs:Class' : kind === 'Property' ? 'rdf:Property' : 'skos:Concept'; +} + +function termsTurtle(): string { + return LOCAL_TERMS.map(([term, kind, label, comment]) => + `mi:${term}\n a ${termType(kind)} ;\n rdfs:label ${literal(label)}@en ;\n rdfs:comment ${literal(comment)}@en ;\n rdfs:isDefinedBy <${VOCABULARY}> .`).join('\n\n'); +} + /** The whole dictionary as Turtle. `issued` maps a published path to its release date. */ export function renderTurtle(repo: RepoModel, refs: RefIndex, issued: Map = new Map()): string { const header = PREFIXES.map(([p, ns]) => `@prefix ${p}: <${ns}> .`).join('\n'); - const terms = LOCAL_TERMS.map(([term, kind, label, comment]) => - `mi:${term}\n a rdfs:${kind === 'Class' ? 'Class' : 'Property'} ;\n rdfs:label ${literal(label)}@en ;\n rdfs:comment ${literal(comment)}@en ;\n rdfs:isDefinedBy <${NS}> .`).join('\n\n'); - const entries = repo.published .filter((f) => f.doc) .sort((a, b) => a.stem.localeCompare(b.stem, 'en')) @@ -206,10 +236,65 @@ ${header} # ---------------------------------------------------------------- local terms (self-describing) -${terms} +${termsTurtle()} # ---------------------------------------------------------------- entries ${entries} `; } + +// ------------------------------------------------------------------ the mi: vocabulary at /ns (#135) + +const VOCABULARY_TITLE = 'material-identity vocabulary (mi:) — proof of concept'; +const VOCABULARY_DESCRIPTION = 'PROOF OF CONCEPT. The local terms of the material-identity dictionary and of its DPP content specifications. Not a vocabulary of the European Union or of CEN; nothing here is normative.'; + +/** The mi: terms as their own Turtle document, served at /ns for `Accept: text/turtle`. */ +export function renderVocabularyTurtle(): string { + const prefixes: Array<[string, string]> = [...PREFIXES, ['owl', 'http://www.w3.org/2002/07/owl#'], ['adms', ADMS]]; + return `# The mi: vocabulary (#135). Derived from the term list in scripts/lib/rdf.ts, like /dictionary.ttl. + +${prefixes.map(([p, ns]) => `@prefix ${p}: <${ns}> .`).join('\n')} + +<${VOCABULARY}> + a owl:Ontology ; + dcterms:title ${literal(VOCABULARY_TITLE)}@en ; + dcterms:description ${literal(VOCABULARY_DESCRIPTION)}@en ; + adms:status mi:ProofOfConcept ; + dcterms:license <${CC0}> ; + dcterms:publisher . + +${termsTurtle()} +`; +} + +/** The same vocabulary as JSON-LD (the default representation of /ns), inline context only. */ +export function renderVocabularyJsonLd(): string { + const graph: Array> = [ + { + '@id': VOCABULARY, + '@type': 'owl:Ontology', + 'dcterms:title': { '@value': VOCABULARY_TITLE, '@language': 'en' }, + 'dcterms:description': { '@value': VOCABULARY_DESCRIPTION, '@language': 'en' }, + 'adms:status': { '@id': 'mi:ProofOfConcept' }, + 'dcterms:license': { '@id': CC0 }, + 'dcterms:publisher': { '@id': 'https://material-identity.eu/' }, + }, + ...LOCAL_TERMS.map(([term, kind, label, comment]) => ({ + '@id': `mi:${term}`, + '@type': termType(kind), + 'rdfs:label': { '@value': label, '@language': 'en' }, + 'rdfs:comment': { '@value': comment, '@language': 'en' }, + 'rdfs:isDefinedBy': { '@id': VOCABULARY }, + })), + ]; + const context: Record = Object.fromEntries([ + ...PREFIXES, ['owl', 'http://www.w3.org/2002/07/owl#'], ['adms', ADMS], + ]); + return `${JSON.stringify({ '@context': context, '@graph': graph }, null, 2)}\n`; +} + +/** The term list for the HTML page — read-only view of the one source. */ +export function vocabularyTerms(): ReadonlyArray { + return LOCAL_TERMS; +} diff --git a/scripts/lib/render.ts b/scripts/lib/render.ts index f7335d7..1f643d8 100644 --- a/scripts/lib/render.ts +++ b/scripts/lib/render.ts @@ -7,6 +7,7 @@ import { CANONICAL_BASE, DEF_PREFIX, type RepoFile, type RepoModel } from './repo.ts'; import { citation } from './cite.ts'; import { graphFigure } from './graph.ts'; +import type { HostedSpec, SpecRole, SpecSummary } from './spec.ts'; export function esc(s: unknown): string { return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); @@ -150,10 +151,12 @@ function pageShell( title: string, canonicalPath: string, body: string, - options: { alternateJson?: string; alternateCsl?: string; rssFeed?: boolean; nav?: string; description?: string } = {}, + options: { alternateJson?: string; alternateCsl?: string; alternateLd?: string; rssFeed?: boolean; nav?: string; description?: string } = {}, ): string { const canonicalUrl = `${CANONICAL_BASE}${canonicalPath}`; - const alternate = (options.alternateJson === undefined + const alternate = (options.alternateLd === undefined + ? '' + : `\n`) + (options.alternateJson === undefined ? '' : `\n`) + (options.alternateCsl === undefined ? '' @@ -185,7 +188,7 @@ ${body} @@ -713,3 +716,194 @@ ${sections || '

No examples in this build.

'}`; description: 'Content specifications that use the material-identity dictionary, shown as reference examples. They are not part of the dictionary.', }); } + +// ------------------------------------------------------------------ hosted specs, catalogue, vocabulary (#135) + +/** + * The proof-of-concept banner on every page about a hosted specification, the catalogue and the + * vocabulary. In both languages the specifications themselves carry, and never dismissible. + */ +export function pocBanner(): string { + return ``; +} + +const localPath = (iri: string | undefined): string | undefined => + iri !== undefined && iri.startsWith(`${CANONICAL_BASE}/`) ? iri.slice(CANONICAL_BASE.length) : undefined; + +/** A link to an IRI: site-relative when it is ours, external otherwise; entries get their label. */ +function iriLink(iri: string | undefined, refs: RefIndex, label?: string): string { + if (iri === undefined) return ''; + if (iri.startsWith(DEF_PREFIX)) return refs.link(iri); + const local = localPath(iri); + return local === undefined ? externalLink(iri) : `${esc(label ?? iri)}`; +} + +const LICENCE_LABEL: Record = { 'http://creativecommons.org/publicdomain/zero/1.0/': 'CC0 1.0' }; + +/** One hosted specification's page: the human representation of `/spec//`. */ +export function renderSpecPage(spec: HostedSpec, summary: SpecSummary, refs: RefIndex): string { + const path = `/spec/${spec.act}/${spec.version}`; + const title = summary.title.en ?? `${spec.act} ${spec.version}`; + const otherTitles = Object.entries(summary.title).filter(([code]) => code !== 'en' && code !== '') + .sort(([a], [b]) => a.localeCompare(b, 'en')) + .map(([code, t]) => `

${esc(t)}

`).join('\n'); + const licence = summary.license === undefined ? '' : `${esc(LICENCE_LABEL[summary.license] ?? summary.license)}`; + const status = summary.status === undefined ? '' : `${iriLink(summary.status, refs, summary.status.replace(/^.*#/, 'mi:'))}`; + + const files = spec.files.map((f) => ` +${esc(f.role === 'dcat' ? 'DCAT record (merged into the catalogue)' : FILE_LABEL[f.role])} +${esc(localPath(f.url) ?? f.url)} +${esc(f.mediaType)} +${esc(f.sha256)} +`).join('\n'); + + const pins = summary.pins.map((p) => `${refs.link(p.url)}${esc(p.url)}${esc(p.sha256 ?? '')}`).join('\n'); + const members = summary.memberships.map((m) => ` +${m.path.map(esc).join('.')} +${m.entry === undefined ? '' : iriLink(m.entry, refs)} +${m.mandatory ? 'mandatory' : 'optional'}${m.condition === undefined ? '' : `
${esc(m.condition)}`} +${m.accessCategory === undefined ? '' : iriLink(m.accessCategory, refs)} +`).join('\n'); + + const body = `${pocBanner()} +

${esc(title)}

+${otherTitles} + +${row('Specification IRI', `${esc(spec.iri)}`)} +${row('Act', `${esc(spec.act)}`)} +${row('Version', `${esc(summary.version ?? spec.version)}`)} +${row('Issued', esc(summary.issued ?? ''))} +${row('Status', status)} +${row('Licence', licence)} +${row('Version series', iriLink(summary.series, refs))} +${row('Conforms to', esc(summary.conformsTo ?? ''))} +${row('DPP schema version', summary.dppSchemaVersion === undefined ? '' : `${esc(summary.dppSchemaVersion)}`)} +${row('Dictionary release', summary.releases.map((r) => `${esc(r.version ?? r.iri)}`).join(', '))} +
+${Object.keys(summary.description).length === 0 ? '' : `

Description

${langMapHtml(summary.description)}
`} + +

Files

+

Hosted byte for byte as material-identity/schema generated them, and never changed: a change is a new version at a new IRI. The SHA-256 is over the served bytes.

+
+ + +${files} + +
RepresentationURLMedia typeSHA-256
+ +

Pinned dictionary entries

+

Each pin names the bytes of one entry by digest. The build of this site checks that every pinned entry is published here and still serves exactly those bytes.

+
+ + +${pins} + +
EntryIRISHA-256
+ +

Memberships

+
+ + +${members} + +
ElementDictionary entryMandatoryAccess
`; + + return pageShell(`${title} — ${spec.version}`, path, body, { + alternateLd: `${path}/index.jsonld`, + description: `Proof of concept: DPP content specification ${spec.act} ${spec.version}, hosted by the material-identity dictionary. Not a content specification of the European Union.`, + }); +} + +const FILE_LABEL: Record, string> = { + spec: 'Specification (JSON-LD, SHACL)', + context: 'Passport @context', + compressed: 'JSON Schema, compressed form', + full: 'JSON Schema, full form', +}; + +/** `/spec/`: every hosted version of one act. */ +export function renderSeriesPage(act: string, versions: Array<{ spec: HostedSpec; summary: SpecSummary }>): string { + const rows = versions.map(({ spec, summary }) => ` +${esc(spec.version)} +${esc(summary.title.en ?? '')} +${esc(summary.issued ?? '')} +`).join('\n'); + const body = `${pocBanner()} +

${esc(act)} — content specifications

+

Every hosted version of the ${esc(act)} content specification. A passport always names one exact version; there is no "latest" alias to pin.

+
+ + +${rows} + +
VersionTitleIssued
+

Machine-readable: JSON-LD (DCAT dataset series) · all specifications and dictionary releases in the catalogue.

`; + return pageShell(`${act} — content specifications`, `/spec/${act}`, body, { + alternateLd: `/spec/${act}/index.jsonld`, + description: `Proof of concept: all hosted versions of the ${act} DPP content specification.`, + }); +} + +/** `/catalog`: the DCAT catalogue as a page. */ +export function renderCatalogPage(releases: Array<{ tag: string; date: string }>, specs: Array<{ spec: HostedSpec; summary: SpecSummary }>): string { + const releaseRows = [...releases].sort((a, b) => a.tag.localeCompare(b.tag, 'en')).map((r) => ` +${esc(r.tag)} +${esc(r.date)} +`).join('\n'); + const specRows = specs.map(({ spec, summary }) => ` +${esc(summary.title.en ?? spec.iri)} +${esc(spec.act)} +${esc(spec.version)} +${esc(summary.issued ?? '')} +`).join('\n'); + const body = `${pocBanner()} +

Catalogue

+

The dictionary releases and the DPP content specifications this site hosts, as a DCAT catalogue. Machine-readable: JSON-LD.

+

Content specifications

+${specRows === '' ? '

No content specification is hosted yet.

' : `
+ + +${specRows} + +
SpecificationActVersionIssued
`} +
+

Dictionary releases

+${releaseRows === '' ? '

No release in this build.

' : `
+ + +${releaseRows} + +
ReleaseDate
`} +
`; + return pageShell('Catalogue', '/catalog', body, { + alternateLd: '/catalog/index.jsonld', + description: 'Proof of concept: DCAT catalogue of the material-identity dictionary releases and hosted DPP content specifications.', + }); +} + +/** `/ns`: the mi: vocabulary, one anchor per term so `…/ns#` lands on its definition. */ +export function renderVocabularyPage(terms: ReadonlyArray): string { + const rows = terms.map(([term, kind, label, comment]) => ` +mi:${esc(term)} +${esc(kind)} +${esc(label)} +${esc(comment)} +`).join('\n'); + const body = `${pocBanner()} +

Vocabulary

+

The local terms of this dictionary and of the content specifications it hosts, in the namespace https://material-identity.eu/ns#. Standard vocabularies (SKOS, DCTerms, DCAT, SHACL, QUDT) are used wherever one fits; a term is defined here only when none does.

+

Machine-readable: Turtle · JSON-LD. Both are also served at /ns itself, by content negotiation.

+
+ + +${rows} + +
TermKindLabelDefinition
`; + return pageShell('Vocabulary', '/ns', body, { + alternateLd: '/ns/index.jsonld', + description: 'Proof of concept: the mi: vocabulary of the material-identity dictionary and its DPP content specifications.', + }); +} diff --git a/scripts/lib/spec.ts b/scripts/lib/spec.ts new file mode 100644 index 0000000..31be66e --- /dev/null +++ b/scripts/lib/spec.ts @@ -0,0 +1,312 @@ +/** + * Hosted DPP content specifications (#135; schema repo docs/design/content-spec-jsonld.md). + * + * material-identity/schema GENERATES a specification; this repo only HOSTS it, byte-identical. + * The source tree is `spec///`, add-only like published/ (check 1), holding exactly + * the files the hand-over carries. The build copies them under the names the Worker maps the + * canonical IRIs onto, so nothing here ever rewrites a byte. + * + * Reading is through oxigraph — already a dependency for the Turtle tests — with no document + * loader: a remote `@context` fails to load, which is exactly the design's rule that a spec's own + * context is inline (§2.3), so the digest covers its complete meaning. + */ +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import oxigraph from 'oxigraph'; +import { CANONICAL_BASE, type ValidationIssue } from './repo.ts'; + +export const SPEC_ROOT = 'spec'; +export const ACT_PATTERN = /^[a-z][a-z0-9-]*$/; +export const VERSION_PATTERN = /^v\d+\.\d+\.\d+$/; + +const SH = 'http://www.w3.org/ns/shacl#'; +const MI = 'https://material-identity.eu/ns#'; +const DCAT = 'http://www.w3.org/ns/dcat#'; +const DCTERMS = 'http://purl.org/dc/terms/'; +const SPDX = 'http://spdx.org/rdf/terms#'; +const ADMS = 'http://www.w3.org/ns/adms#'; + +export const PREFIXES = `PREFIX sh: <${SH}> PREFIX mi: <${MI}> PREFIX dcat: <${DCAT}> PREFIX dcterms: <${DCTERMS}> PREFIX spdx: <${SPDX}> PREFIX adms: <${ADMS}> `; + +export type SpecRole = 'spec' | 'context' | 'dcat' | 'compressed' | 'full'; + +/** One handed-over file and, if it is served on its own, the URL and origin name it is served as. */ +export interface SpecFile { + role: SpecRole; + /** Name in the hand-over and in `spec///`. */ + source: string; + /** Name below `site/spec///`; undefined for the DCAT record, which /catalog serves. */ + served?: string; + /** Canonical URL; for the DCAT record, the catalogue it is merged into. */ + url: string; + mediaType: string; + bytes: Buffer; + sha256: string; +} + +export interface HostedSpec { + act: string; + version: string; + /** `https://material-identity.eu/spec//` */ + iri: string; + /** `spec//` */ + relDir: string; + files: SpecFile[]; + /** The specification's RDF (its own document only). */ + graph: oxigraph.Store; + /** The DCAT record as JSON (merged into /catalog as written) and as RDF (checked). */ + dcatJson: Record; + dcatGraph: oxigraph.Store; +} + +export interface SpecLoad { + specs: HostedSpec[]; + issues: ValidationIssue[]; +} + +export const specIri = (act: string, version: string): string => `${CANONICAL_BASE}/spec/${act}/${version}`; + +/** The hand-over, file by file (design §7.2 step 3), and where each one is served. */ +function layout(act: string, iri: string): Array> { + return [ + { role: 'spec', source: `${act}.jsonld`, served: 'index.jsonld', url: iri, mediaType: 'application/ld+json' }, + { role: 'context', source: `${act}.context.jsonld`, served: 'context.jsonld', url: `${iri}/context.jsonld`, mediaType: 'application/ld+json' }, + { role: 'dcat', source: `${act}.dcat.jsonld`, url: `${CANONICAL_BASE}/catalog`, mediaType: 'application/ld+json' }, + { role: 'compressed', source: 'compressed.schema.json', served: 'compressed.schema.json', url: `${iri}/compressed.schema.json`, mediaType: 'application/schema+json' }, + { role: 'full', source: 'full.schema.json', served: 'full.schema.json', url: `${iri}/full.schema.json`, mediaType: 'application/schema+json' }, + ]; +} + +/** Parse JSON-LD with no document loader: remote contexts fail, as they must (design §2.3). */ +export function parseJsonLd(text: string, base: string): oxigraph.Store { + const store = new oxigraph.Store(); + store.load(text, { format: 'application/ld+json', base_iri: base }); + return store; +} + +const dirs = (abs: string): string[] => readdirSync(abs).sort(); +const isDir = (abs: string): boolean => statSync(abs).isDirectory(); + +/** + * Every `spec///`, sorted by act then version. Structural problems (names, missing or + * extra files, unparseable JSON or JSON-LD) are returned as issues and the directory is skipped. + */ +export function loadSpecs(root: string): SpecLoad { + const specs: HostedSpec[] = []; + const issues: ValidationIssue[] = []; + const check = 'spec layout'; + const base = join(root, SPEC_ROOT); + if (!existsSync(base)) return { specs, issues }; + + for (const act of dirs(base)) { + const actAbs = join(base, act); + if (!isDir(actAbs) || !ACT_PATTERN.test(act)) { + issues.push({ check, file: `${SPEC_ROOT}/${act}`, message: `not an act directory (expected ${ACT_PATTERN})` }); + continue; + } + for (const version of dirs(actAbs).sort(compareVersions)) { + const relDir = `${SPEC_ROOT}/${act}/${version}`; + const abs = join(actAbs, version); + if (!isDir(abs) || !VERSION_PATTERN.test(version)) { + issues.push({ check, file: relDir, message: `not a version directory (expected ${VERSION_PATTERN})` }); + continue; + } + const iri = specIri(act, version); + const expected = layout(act, iri); + const present = new Set(dirs(abs)); + const before = issues.length; + for (const f of expected) { + if (!present.has(f.source)) issues.push({ check, file: `${relDir}/${f.source}`, message: 'missing from the hand-over' }); + } + for (const name of present) { + if (!expected.some((f) => f.source === name)) issues.push({ check, file: `${relDir}/${name}`, message: 'not part of a content-specification hand-over' }); + } + if (issues.length > before) continue; + + const files: SpecFile[] = expected.map((f) => { + const bytes = readFileSync(join(abs, f.source)); + return { ...f, bytes, sha256: createHash('sha256').update(bytes).digest('hex') }; + }); + const parsed = new Map(); + for (const f of files) { + const text = f.bytes.toString('utf8'); + try { + const json: unknown = JSON.parse(text); + parsed.set(f.role, { json, graph: f.source.endsWith('.jsonld') ? parseJsonLd(text, f.url) : undefined }); + } catch (err) { + issues.push({ check, file: `${relDir}/${f.source}`, message: `does not parse as ${f.source.endsWith('.jsonld') ? 'JSON-LD' : 'JSON'}: ${(err as Error).message}` }); + } + } + if (issues.length > before) continue; + const dcatJson = parsed.get('dcat')!.json; + if (!dcatJson || typeof dcatJson !== 'object' || Array.isArray(dcatJson)) { + issues.push({ check, file: `${relDir}/${act}.dcat.jsonld`, message: 'the DCAT record is not a JSON object' }); + continue; + } + specs.push({ + act, version, iri, relDir, files, + graph: parsed.get('spec')!.graph!, + dcatJson: dcatJson as Record, + dcatGraph: parsed.get('dcat')!.graph!, + }); + } + } + return { specs, issues }; +} + +/** `v0.0.10` after `v0.0.9`: numeric, not lexical. Non-versions sort after, by name. */ +export function compareVersions(a: string, b: string): number { + const parts = (v: string): number[] | undefined => (VERSION_PATTERN.test(v) ? v.slice(1).split('.').map(Number) : undefined); + const pa = parts(a); + const pb = parts(b); + if (!pa || !pb) return pa ? -1 : pb ? 1 : a.localeCompare(b, 'en'); + for (let i = 0; i < 3; i++) if (pa[i] !== pb[i]) return pa[i] - pb[i]; + return 0; +} + +// ------------------------------------------------------------------ reading a spec + +type Value = oxigraph.NamedNode | oxigraph.BlankNode | oxigraph.Literal; +type Row = Map; + +export function select(store: oxigraph.Store, where: string): Row[] { + return store.query(`${PREFIXES} SELECT * WHERE { ${where} }`) as unknown as Row[]; +} + +export function ask(store: oxigraph.Store, where: string): boolean { + return store.query(`${PREFIXES} ASK { ${where} }`) as unknown as boolean; +} + +/** Literal values of one predicate on one subject, keyed by language tag ('' when untagged). */ +function langValues(store: oxigraph.Store, subject: string, predicate: string): Record { + const out: Record = {}; + const rows = select(store, `<${subject}> ${predicate} ?v`).sort((a, b) => a.get('v')!.value.localeCompare(b.get('v')!.value, 'en')); + for (const row of rows) { + const v = row.get('v')!; + const key = v.termType === 'Literal' ? (v as oxigraph.Literal).language : ''; + if (out[key] === undefined) out[key] = v.value; + } + return out; +} + +function one(store: oxigraph.Store, subject: string, predicate: string): string | undefined { + return select(store, `<${subject}> ${predicate} ?v`).map((r) => r.get('v')!.value).sort()[0]; +} + +export interface Pin { + /** `dcat:downloadURL` — the pinned entry IRI. */ + url: string; + sha256?: string; +} + +/** `dcterms:requires` → distributions (pinned entries) of the subject. */ +export function distributionPins(store: oxigraph.Store, subject: string, predicate = 'dcterms:requires'): Pin[] { + const rows = select(store, `<${subject}> ${predicate} ?d . ?d dcat:downloadURL ?u . OPTIONAL { ?d spdx:checksum ?c . ?c spdx:checksumValue ?sum }`); + const pins = new Map(); + for (const row of rows) { + const url = row.get('u')!.value; + const sha256 = row.get('sum')?.value; + if (!pins.has(url) || (sha256 !== undefined && pins.get(url)!.sha256 === undefined)) pins.set(url, { url, sha256 }); + } + return [...pins.values()].sort((a, b) => a.url.localeCompare(b.url, 'en')); +} + +export interface Membership { + /** Dotted element-id path from the root, e.g. `nominalDimensions.nominalThickness`. */ + path: string[]; + /** The dictionary entry: `sh:path` when it is an IRI, else the list item's `mi:itemType`. */ + entry?: string; + mandatory: boolean; + accessCategory?: string; + condition?: string; +} + +export interface SpecSummary { + title: Record; + description: Record; + version?: string; + issued?: string; + license?: string; + status?: string; + series?: string; + conformsTo?: string; + dppSchemaVersion?: string; + passportContext?: string; + releases: Array<{ iri: string; version?: string }>; + pins: Pin[]; + memberships: Membership[]; +} + +/** What the HTML, the series index and the catalogue say about a spec — read from its RDF. */ +export function summarize(spec: HostedSpec): SpecSummary { + const g = spec.graph; + const s = spec.iri; + const releases = select(g, `<${s}> dcterms:requires ?r . ?r a dcat:Dataset . OPTIONAL { ?r dcat:version ?v }`) + .map((r) => ({ iri: r.get('r')!.value, version: r.get('v')?.value })) + .sort((a, b) => a.iri.localeCompare(b.iri, 'en')); + return { + title: langValues(g, s, 'dcterms:title'), + description: langValues(g, s, 'dcterms:description'), + version: one(g, s, 'dcat:version'), + issued: one(g, s, 'dcterms:issued'), + license: one(g, s, 'dcterms:license'), + status: one(g, s, 'adms:status'), + series: one(g, s, 'dcat:isVersionOf'), + conformsTo: one(g, s, 'dcterms:conformsTo'), + dppSchemaVersion: one(g, s, 'mi:dppSchemaVersion'), + passportContext: one(g, s, 'mi:passportContext'), + releases: releases.filter((r, i) => i === 0 || r.iri !== releases[i - 1].iri), + pins: distributionPins(g, s), + memberships: memberships(g, s), + }; +} + +/** + * Memberships depth-first in `sh:order` (design §3.2: array position is not significant in RDF). + * A membership is any property shape with an `mi:elementId`; its children sit under its `sh:node`. + */ +function memberships(g: oxigraph.Store, root: string): Membership[] { + const rows = select(g, ` + { <${root}> sh:property ?p . BIND(<${root}> AS ?parent) } UNION { ?parent sh:node ?n . ?n sh:property ?p } + ?p mi:elementId ?id . + OPTIONAL { ?p sh:path ?path FILTER(isIRI(?path)) } + OPTIONAL { ?p mi:itemType ?item } + OPTIONAL { ?p sh:order ?order } + OPTIONAL { ?p sh:minCount ?min } + OPTIONAL { ?p mi:accessCategory ?access } + OPTIONAL { ?p mi:condition ?cond FILTER(langMatches(lang(?cond), "en")) }`); + const key = (t: Value): string => `${t.termType}:${t.value}`; + interface Node { key: string; id: string; order: number; m: Omit } + const children = new Map(); + const seen = new Set(); + for (const r of rows) { + const k = key(r.get('p')!); + if (seen.has(k)) continue; + seen.add(k); + const parent = key(r.get('parent')!); + const node: Node = { + key: k, + id: r.get('id')!.value, + order: Number(r.get('order')?.value ?? Number.MAX_SAFE_INTEGER), + m: { + entry: r.get('path')?.value ?? r.get('item')?.value, + mandatory: Number(r.get('min')?.value ?? 0) >= 1, + accessCategory: r.get('access')?.value, + condition: r.get('cond')?.value, + }, + }; + children.set(parent, [...(children.get(parent) ?? []), node]); + } + const out: Membership[] = []; + const walk = (parentKey: string, path: string[], visiting: Set): void => { + const list = [...(children.get(parentKey) ?? [])].sort((a, b) => a.order - b.order || a.id.localeCompare(b.id, 'en')); + for (const n of list) { + if (visiting.has(n.key)) continue; // a shape graph may be cyclic; a table may not + out.push({ path: [...path, n.id], ...n.m }); + walk(n.key, [...path, n.id], new Set([...visiting, n.key])); + } + }; + walk(`NamedNode:${root}`, [], new Set()); + return out; +} diff --git a/scripts/lib/styles.css b/scripts/lib/styles.css index 1767098..575ac4b 100644 --- a/scripts/lib/styles.css +++ b/scripts/lib/styles.css @@ -19,6 +19,7 @@ --chip-neutral-bg: #e7efea; --chip-neutral-fg: #2f4a3b; --chip-tier-bg: #ffedd5; --chip-tier-fg: #c2410c; --dead-bg: #fbe4e4; --dead-edge: #b3423d; + --poc-bg: #fef3c7; --poc-edge: #b45309; } @media (prefers-color-scheme: dark) { :root { @@ -38,6 +39,7 @@ --chip-neutral-bg: #1c3529; --chip-neutral-fg: #d2e3d9; --chip-tier-bg: #6a300d; --chip-tier-fg: #fed7aa; --dead-bg: #4a1d1b; --dead-edge: #e07a75; + --poc-bg: #4a3208; --poc-edge: #f0b44c; } } @@ -205,3 +207,7 @@ ul.graph-key li { display: flex; align-items: center; gap: 0.45rem; } } @media (prefers-reduced-motion: no-preference) { a { transition: text-decoration-thickness 120ms; } } @media print { .mast, footer, .btn { display: none; } } + +/* Proof-of-concept banner (#135): on every page about a hosted specification, the catalogue and /ns. */ +.banner.poc { background: var(--poc-bg); border-color: var(--poc-edge); } +.banner.poc p { margin: 0.25rem 0; } diff --git a/test/checks.test.ts b/test/checks.test.ts index aeb83cd..ac1decc 100644 --- a/test/checks.test.ts +++ b/test/checks.test.ts @@ -90,7 +90,7 @@ test('check 5 — isDefinedBy and replaces are exempt from pinning', () => { test('runChecks aggregates load errors and all checks 1–6', () => { const results = runChecks(load('red-yaml')); - assert.equal(results.length, 9); + assert.equal(results.length, 10); const loadResult = results.find((r) => r.name === 'load'); assert.ok(loadResult && loadResult.issues.length === 1); const failing = results.filter((r) => r.issues.length > 0); diff --git a/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json b/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json new file mode 100644 index 0000000..4dab70b --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/compressed.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://material-identity.eu/spec/demo/v0.0.1/compressed.schema.json", + "$comment": "SPDX-License-Identifier: CC0-1.0. PROOF OF CONCEPT test fixture.", + "type": "object", + "required": ["mechanicalProperties"], + "properties": { + "mechanicalProperties": { "type": "object", "required": ["maxPressure"], "properties": { "maxPressure": { "type": "number" } }, "additionalProperties": false }, + "steelmakingRoute": { "type": "string", "format": "uri" } + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld new file mode 100644 index 0000000..db597f0 --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.context.jsonld @@ -0,0 +1,16 @@ +{ + "@context": { + "@version": 1.1, + "@protected": true, + "def": "https://material-identity.eu/def/", + "mi": "https://material-identity.eu/ns#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "contentSpecificationIds": { "@id": "@type", "@container": "@set" }, + "dppSchemaVersion": { "@id": "mi:dppSchemaVersion", "@type": "@id" }, + "mechanicalProperties": { + "@id": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", + "@context": { "maxPressure": { "@id": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", "@type": "xsd:float" } } + }, + "steelmakingRoute": { "@id": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", "@type": "@id" } + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld new file mode 100644 index 0000000..d896773 --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.dcat.jsonld @@ -0,0 +1,56 @@ +{ + "@context": { + "dcat": "http://www.w3.org/ns/dcat#", + "dcterms": "http://purl.org/dc/terms/", + "adms": "http://www.w3.org/ns/adms#", + "spdx": "http://spdx.org/rdf/terms#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "mi": "https://material-identity.eu/ns#" + }, + "@id": "https://material-identity.eu/catalog", + "@type": "dcat:Catalog", + "dcat:dataset": { + "@id": "https://material-identity.eu/spec/demo/v0.0.1", + "@type": "dcat:Dataset", + "dcterms:title": { "@value": "Demo content specification — test fixture (PoC)", "@language": "en" }, + "dcterms:description": { "@value": "PROOF OF CONCEPT. Not a content specification of the European Union.", "@language": "en" }, + "adms:status": { "@id": "mi:ProofOfConcept" }, + "dcat:version": "v0.0.1", + "dcat:isVersionOf": { "@id": "https://material-identity.eu/spec/demo" }, + "dcterms:issued": { "@value": "2026-11-02", "@type": "xsd:date" }, + "dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }, + "dcterms:conformsTo": { "@id": "urn:cen:en:18223:2026" }, + "dcterms:requires": { "@id": "https://material-identity.eu/release/v2026.08.28" }, + "dcat:distribution": [ + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#jsonld", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1" }, + "dcat:mediaType": "application/ld+json", + "dcterms:conformsTo": { "@id": "http://www.w3.org/ns/shacl#" }, + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "c90dfd49a794e177b19c1559427c1c504ba58016c49b9a01ef353565f86a4bd7" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#context", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/context.jsonld" }, + "dcat:mediaType": "application/ld+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "64c71906fd2eeb51a2e6b54480dc9ac530ceed17f949dd483da3ea7c8d6f97b4" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#compressed", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/compressed.schema.json" }, + "dcat:mediaType": "application/schema+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "eb16cf4331e090f6813d873790ff70a3445afbd2ea32e4de5abdbe0ff35546e4" } + }, + { + "@id": "https://material-identity.eu/spec/demo/v0.0.1#full", + "@type": "dcat:Distribution", + "dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/full.schema.json" }, + "dcat:mediaType": "application/schema+json", + "spdx:checksum": { "spdx:algorithm": { "@id": "spdx:checksumAlgorithm_sha256" }, "spdx:checksumValue": "bb27ef5ee80e4abcc1f38a6e1a09df0d4e57185ae73fc8b793a7a5dccad93a1d" } + } + ] + } +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld b/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld new file mode 100644 index 0000000..50fdcde --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/demo.jsonld @@ -0,0 +1,90 @@ +{ + "@context": { + "@version": 1.1, + "sh": "http://www.w3.org/ns/shacl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dcterms": "http://purl.org/dc/terms/", + "dcat": "http://www.w3.org/ns/dcat#", + "adms": "http://www.w3.org/ns/adms#", + "spdx": "http://spdx.org/rdf/terms#", + "mi": "https://material-identity.eu/ns#", + "def": "https://material-identity.eu/def/", + "s": "https://material-identity.eu/spec/demo/v0.0.1#", + "title": { "@id": "dcterms:title", "@container": "@language" }, + "description": { "@id": "dcterms:description", "@container": "@language" }, + "pin": { "@id": "dcterms:requires", "@container": "@set" }, + "entry": { "@id": "dcat:downloadURL", "@type": "@id" }, + "checksum": "spdx:checksum", + "algorithm": { "@id": "spdx:algorithm", "@type": "@id" }, + "sha256": "spdx:checksumValue", + "property": { "@id": "sh:property", "@container": "@set" }, + "path": { "@id": "sh:path", "@type": "@id" }, + "datatype": { "@id": "sh:datatype", "@type": "@id" }, + "nodeKind": { "@id": "sh:nodeKind", "@type": "@id" }, + "minCount": "sh:minCount", + "maxCount": "sh:maxCount", + "order": "sh:order", + "node": "sh:node", + "closed": "sh:closed", + "elementId": "mi:elementId", + "accessCategory": { "@id": "mi:accessCategory", "@type": "@vocab" }, + "granularity": { "@id": "mi:granularity", "@type": "@id" }, + "condition": { "@id": "mi:condition", "@container": "@language" }, + "authorityOnly": "https://material-identity.eu/def/7a1e2c3d-4b5f-4a6e-9c8d-1f2e3d4c5b6a" + }, + "@id": "https://material-identity.eu/spec/demo/v0.0.1", + "@type": ["mi:ContentSpecification", "sh:NodeShape", "rdfs:Class"], + "title": { + "en": "Demo content specification — test fixture (PoC)", + "de": "Demo-Inhaltsspezifikation — Testdatensatz (PoC)" + }, + "description": { + "en": "PROOF OF CONCEPT. A test fixture of the dictionary repository. Not a content specification of the European Union.", + "de": "PROOF OF CONCEPT. Ein Testdatensatz des Wörterbuch-Repositorys. Keine Inhaltsspezifikation der Europäischen Union." + }, + "adms:status": { "@id": "mi:ProofOfConcept" }, + "dcat:version": "v0.0.1", + "dcat:isVersionOf": { "@id": "https://material-identity.eu/spec/demo" }, + "dcterms:issued": { "@value": "2026-11-02", "@type": "xsd:date" }, + "dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }, + "dcterms:rights": "SPDX-License-Identifier: CC0-1.0", + "dcterms:publisher": { "@id": "https://material-identity.eu/" }, + "dcterms:conformsTo": { "@id": "urn:cen:en:18223:2026" }, + "mi:act": "demo", + "mi:dppSchemaVersion": { "@id": "https://material-identity.org/schemas/dpp/v0.0.3" }, + "mi:passportContext": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/context.jsonld" }, + + "pin": [ + { "@id": "https://material-identity.eu/release/v2026.08.28", "@type": "dcat:Dataset", "dcat:version": "v2026.08.28" }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "cb2f32484b2602aad2a2fbb189722ba140e310927745de7a00e6ff8aefd0722d" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "51a28c90bcf24fc116251ef96216a331f278556bf92935dda49f345971cfe075" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "f136650e084da6b9e55657d0556957f14bcf8b1aed7256fb3af4ba46f7f94476" } }, + { "@type": "dcat:Distribution", "dcat:mediaType": "application/json", "entry": "def:7a1e2c3d-4b5f-4a6e-9c8d-1f2e3d4c5b6a", + "checksum": { "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "a9ba77a780962cfe6242caedf9d8a33919096d15b234a7a15629269c3c3a6908" } } + ], + + "closed": true, + "sh:ignoredProperties": { "@list": [{ "@id": "rdf:type" }] }, + "property": [ + { "path": "mi:dppSchemaVersion", "sh:hasValue": { "@id": "https://material-identity.org/schemas/dpp/v0.0.3" }, "minCount": 1, "maxCount": 1 }, + + { "@id": "s:steelmakingRoute", "elementId": "steelmakingRoute", "order": 2, + "path": "def:4eae703a-fa15-4118-8bc4-7926a22a12fb", "nodeKind": "sh:IRI", "maxCount": 1, + "accessCategory": "authorityOnly", "granularity": "mi:batch", + "condition": { "en": "Mandatory where the route is known.", "de": "Verpflichtend, wenn die Route bekannt ist." } }, + + { "@id": "s:mechanicalProperties", "elementId": "mechanicalProperties", "order": 1, + "path": "def:2f3de2bb-0588-4513-bfc3-41d021815a81", "minCount": 1, "maxCount": 1, + "accessCategory": "authorityOnly", "granularity": "mi:batch", + "node": { "@id": "s:MechanicalProperties", "@type": "sh:NodeShape", "closed": true, "property": [ + { "@id": "s:mechanicalProperties.maxPressure", "elementId": "maxPressure", "order": 1, + "path": "def:c38a85eb-1a37-416d-ab21-7ddcc599754d", "datatype": "xsd:float", "minCount": 1, "maxCount": 1, + "accessCategory": "authorityOnly" } + ] } } + ] +} diff --git a/test/fixtures/green/spec/demo/v0.0.1/full.schema.json b/test/fixtures/green/spec/demo/v0.0.1/full.schema.json new file mode 100644 index 0000000..dae232c --- /dev/null +++ b/test/fixtures/green/spec/demo/v0.0.1/full.schema.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://material-identity.eu/spec/demo/v0.0.1/full.schema.json", + "$comment": "SPDX-License-Identifier: CC0-1.0. PROOF OF CONCEPT test fixture.", + "type": "object", + "required": ["elements"], + "properties": { "elements": { "type": "array", "maxItems": 2 } } +} diff --git a/test/index-worker.test.ts b/test/index-worker.test.ts index c2b7349..e082db2 100644 --- a/test/index-worker.test.ts +++ b/test/index-worker.test.ts @@ -173,3 +173,84 @@ test('worker fetch handler maps to the origin and stamps headers (stubbed origin globalThis.fetch = realFetch; } }); + +// ------------------------------------------------------------------ content specifications, catalogue, vocabulary (#135) + +test('decide: a spec IRI is JSON-LD by default, HTML for browsers, immutable, cite-as itself', () => { + const ld = decide('/spec/steel/v0.0.3', 'application/ld+json'); + assert.deepEqual(ld, { + originPath: '/spec/steel/v0.0.3/index.jsonld', + headers: { + 'content-type': 'application/ld+json', + 'cache-control': 'public, max-age=31536000, immutable', + vary: 'Accept', + link: '; rel="cite-as"', + }, + }); + assert.equal(decide('/spec/steel/v0.0.3', '*/*').originPath, '/spec/steel/v0.0.3/index.jsonld'); + assert.equal(decide('/spec/steel/v0.0.3', null).originPath, '/spec/steel/v0.0.3/index.jsonld'); + + const html = decide('/spec/steel/v0.0.3', 'text/html,application/xhtml+xml,*/*;q=0.8'); + assert.equal(html.originPath, '/spec/steel/v0.0.3/index.html'); + assert.equal(html.headers['content-type'], 'text/html; charset=utf-8'); + assert.equal(html.headers['cache-control'], 'public, max-age=31536000, immutable'); + assert.equal(html.headers.link, '; rel="canonical", ; rel="cite-as"'); + // multi-digit versions and hyphenated acts are legitimate + assert.equal(decide('/spec/construction-products/v10.20.300', '*/*').originPath, '/spec/construction-products/v10.20.300/index.jsonld'); +}); + +test('decide: a spec\'s sub-resources are typed and immutable, whatever the Accept header', () => { + const ctx = decide('/spec/steel/v0.0.3/context.jsonld', 'text/html'); + assert.deepEqual(ctx, { originPath: '/spec/steel/v0.0.3/context.jsonld', headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=31536000, immutable' } }); + for (const name of ['compressed.schema.json', 'full.schema.json']) { + const schema = decide(`/spec/steel/v0.0.3/${name}`, '*/*'); + assert.deepEqual(schema, { originPath: `/spec/steel/v0.0.3/${name}`, headers: { 'content-type': 'application/schema+json', 'cache-control': 'public, max-age=31536000, immutable' } }); + } +}); + +test('decide: the series index and the catalogue negotiate like a spec but keep a short cache', () => { + for (const [path, dir] of [['/spec/steel', '/spec/steel'], ['/catalog', '/catalog']]) { + assert.deepEqual(decide(path, '*/*'), { originPath: `${dir}/index.jsonld`, headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.deepEqual(decide(path, 'text/html'), { originPath: `${dir}/index.html`, headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.ok(!decide(path, '*/*').headers['cache-control'].includes('immutable'), `${path} gains members, so it must never be immutable`); + } +}); + +test('decide: /ns serves the vocabulary as JSON-LD, Turtle on request, HTML for browsers', () => { + assert.deepEqual(decide('/ns', '*/*'), { originPath: '/ns/index.jsonld', headers: { 'content-type': 'application/ld+json', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.deepEqual(decide('/ns', 'text/turtle'), { originPath: '/ns/index.ttl', headers: { 'content-type': 'text/turtle; charset=utf-8', 'cache-control': 'public, max-age=300', vary: 'Accept' } }); + assert.equal(decide('/ns', 'text/html').originPath, '/ns/index.html'); + assert.equal(decide('/ns', 'text/html, text/turtle;q=0.5').originPath, '/ns/index.html'); // a browser always gets the page +}); + +test('decide: anything that is not exactly a spec shape falls through as before', () => { + for (const p of [ + '/spec', '/spec/', '/spec/steel/', '/spec/Steel', '/spec/1steel', '/spec/steel_x', '/spec/steel/v0.0.3/', '/spec/steel/0.0.3', + '/spec/steel/v0.3', '/spec/steel/latest', '/spec/steel/v0.0.3/steel.dcat.jsonld', '/spec/steel/v0.0.3/index.jsonld', + '/spec/steel/v0.0.3/context.jsonld/x', '/spec/a/b/c/d', '/catalog/', '/catalog/index.jsonld', '/ns/', '/ns/index.ttl', '/nsx', + ]) { + assert.deepEqual(decide(p, 'text/html'), { originPath: p, headers: { 'cache-control': 'public, max-age=300' } }, `${p} must pass through`); + } + // no origin path we map to may start with a dot (#110) + for (const p of ['/spec/steel/v0.0.3', '/spec/steel/v0.0.3/context.jsonld', '/spec/steel', '/catalog', '/ns']) { + for (const accept of ['*/*', 'text/html', 'text/turtle']) assert.ok(!decide(p, accept).originPath.includes('/.')); + } +}); + +test('worker fetch handler serves a spec as JSON-LD from its origin file (stubbed origin)', async () => { + const seen: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request) => { + seen.push(String(input)); + return new Response('{"@id":"x"}', { status: 200, headers: { 'content-type': 'application/octet-stream' } }); + }) as typeof fetch; + try { + const res = await worker.fetch(new Request('https://material-identity.eu/spec/steel/v0.0.3'), { ORIGIN: 'https://origin.example/dictionary' }); + assert.equal(res.status, 200); + assert.equal(res.headers.get('content-type'), 'application/ld+json'); // the origin's guess is overridden + assert.equal(res.headers.get('vary'), 'Accept'); + assert.deepEqual(seen, ['https://origin.example/dictionary/spec/steel/v0.0.3/index.jsonld']); + } finally { + globalThis.fetch = realFetch; + } +}); diff --git a/test/spec.test.ts b/test/spec.test.ts new file mode 100644 index 0000000..6528b05 --- /dev/null +++ b/test/spec.test.ts @@ -0,0 +1,402 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import oxigraph from 'oxigraph'; +import { build, emitSpecs } from '../scripts/build.ts'; +import { checkImmutability, checkSpecs, runChecks } from '../scripts/lib/checks.ts'; +import { buildCatalog, buildSeries, embeddedDataset, findDatasetNode } from '../scripts/lib/dcat.ts'; +import { getReleaseTags } from '../scripts/lib/git.ts'; +import { renderVocabularyJsonLd, renderVocabularyTurtle, vocabularyTerms } from '../scripts/lib/rdf.ts'; +import { loadRepo } from '../scripts/lib/repo.ts'; +import { pocBanner, RefIndex, renderCatalogPage, renderVocabularyPage } from '../scripts/lib/render.ts'; +import { compareVersions, loadSpecs, parseJsonLd } from '../scripts/lib/spec.ts'; + +// Hosted content specifications (#135). The fixture is a small spec authored for these tests +// under test/fixtures/green/spec/ — never under the repo's own spec/, which only ever holds +// hand-overs from material-identity/schema. +const here = dirname(fileURLToPath(import.meta.url)); +const GREEN = join(here, 'fixtures', 'green'); +const REL = join('spec', 'demo', 'v0.0.1'); +const IRI = 'https://material-identity.eu/spec/demo/v0.0.1'; +const SERVED = ['demo.jsonld', 'demo.context.jsonld', 'compressed.schema.json', 'full.schema.json']; + +const sha = (bytes: Buffer | string): string => createHash('sha256').update(bytes).digest('hex'); +const ask = (store: oxigraph.Store, where: string): boolean => store.query(`ASK { ${where} }`) as unknown as boolean; + +/** + * A throwaway copy of the green tree whose spec files `mutate` may rewrite. With `refresh`, the DCAT + * record's checksums are recomputed afterwards, so a test sees only the defect it planted. + */ +function withTree(mutate: (dir: string, root: string) => void, fn: (root: string) => void, refresh = true): void { + const root = mkdtempSync(join(tmpdir(), 'spec-tree-')); + try { + cpSync(GREEN, root, { recursive: true }); + const dir = join(root, REL); + const before = new Map(SERVED.map((n) => [sha(readFileSync(join(dir, n))), n])); + mutate(dir, root); + if (refresh) { + const dcat = join(dir, 'demo.dcat.jsonld'); + let text = readFileSync(dcat, 'utf8'); + for (const [old, name] of before) { + try { text = text.replace(old, sha(readFileSync(join(dir, name)))); } catch { /* file removed by the mutation */ } + } + writeFileSync(dcat, text); + } + fn(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +const edit = (dir: string, name: string, change: (text: string) => string): void => { + writeFileSync(join(dir, name), change(readFileSync(join(dir, name), 'utf8'))); +}; + +const specIssues = (root: string): string[] => checkSpecs(loadSpecs(root), loadRepo(root)).map((i) => `${i.file}: ${i.message}`); + +// ------------------------------------------------------------------ loading + +test('loadSpecs reads the fixture: one spec, five files, each hashed over its exact bytes', () => { + const { specs, issues } = loadSpecs(GREEN); + assert.deepEqual(issues, []); + assert.equal(specs.length, 1); + const [spec] = specs; + assert.equal(spec.iri, IRI); + assert.equal(spec.relDir, 'spec/demo/v0.0.1'); + assert.deepEqual(spec.files.map((f) => f.source), ['demo.jsonld', 'demo.context.jsonld', 'demo.dcat.jsonld', 'compressed.schema.json', 'full.schema.json']); + for (const f of spec.files) assert.equal(f.sha256, sha(readFileSync(join(GREEN, REL, f.source)))); + assert.deepEqual(spec.files.map((f) => f.served), ['index.jsonld', 'context.jsonld', undefined, 'compressed.schema.json', 'full.schema.json']); +}); + +test('a repo without spec/ hosts nothing and is fine', () => { + assert.deepEqual(loadSpecs(join(here, 'fixtures', 'empty')), { specs: [], issues: [] }); +}); + +test('versions sort numerically, not lexically', () => { + assert.deepEqual(['v0.0.10', 'v0.0.9', 'v1.0.0', 'v0.1.0'].sort(compareVersions), ['v0.0.9', 'v0.0.10', 'v0.1.0', 'v1.0.0']); + assert.equal(compareVersions('v1.2.3', 'v1.2.3'), 0); + assert.ok(compareVersions('v1.0.0', 'junk') < 0 && compareVersions('junk', 'v1.0.0') > 0); + assert.ok(compareVersions('a', 'b') < 0); +}); + +test('layout: misnamed directories, missing and extra files, and stray files are refused', () => { + withTree((dir, root) => { + rmSync(join(dir, 'full.schema.json')); + writeFileSync(join(dir, 'notes.md'), 'x'); + mkdirSync(join(root, 'spec', 'Bad_Act')); + mkdirSync(join(root, 'spec', 'demo', 'latest')); + writeFileSync(join(root, 'spec', 'README.md'), 'x'); + }, (root) => { + const issues = loadSpecs(root).issues.map((i) => `${i.file}: ${i.message}`); + assert.ok(issues.includes('spec/demo/v0.0.1/full.schema.json: missing from the hand-over')); + assert.ok(issues.includes('spec/demo/v0.0.1/notes.md: not part of a content-specification hand-over')); + assert.ok(issues.some((i) => i.startsWith('spec/Bad_Act: not an act directory'))); + assert.ok(issues.some((i) => i.startsWith('spec/README.md: not an act directory'))); + assert.ok(issues.some((i) => i.startsWith('spec/demo/latest: not a version directory'))); + assert.equal(loadSpecs(root).specs.length, 0); + }, false); +}); + +test('layout: unparseable JSON, unparseable JSON-LD, a remote @context and a non-object DCAT record are refused', () => { + withTree((dir) => { + writeFileSync(join(dir, 'full.schema.json'), '{ not json'); + }, (root) => assert.match(loadSpecs(root).issues[0].message, /does not parse as JSON:/), false); + + withTree((dir) => { + edit(dir, 'demo.context.jsonld', (t) => t.replace('"@version": 1.1,', '"@version": 1.1, "@base": 42,')); + }, (root) => assert.match(loadSpecs(root).issues[0].message, /does not parse as JSON-LD:/), false); + + // design §2.3: the spec's own context is inline — no loader is configured, so a remote one fails + withTree((dir) => { + edit(dir, 'demo.jsonld', (t) => t.replace(/"@context": \{[\s\S]*?\n \},/, '"@context": "https://example.org/remote.jsonld",')); + }, (root) => { + const [issue] = loadSpecs(root).issues; + assert.equal(issue.file, 'spec/demo/v0.0.1/demo.jsonld'); + assert.match(issue.message, /does not parse as JSON-LD/); + }, false); + + withTree((dir) => writeFileSync(join(dir, 'demo.dcat.jsonld'), '[]'), (root) => { + assert.match(loadSpecs(root).issues[0].message, /the DCAT record is not a JSON object/); + }, false); +}); + +// ------------------------------------------------------------------ the spec check + +test('the green fixture passes the spec check, and runChecks runs it', () => { + assert.deepEqual(specIssues(GREEN), []); + const result = runChecks(loadRepo(GREEN)).find((r) => r.name === 'spec — hosted content specifications'); + assert.deepEqual(result?.issues, []); +}); + +test('spec check: a spec not marked proof of concept, or not CC0, is refused', () => { + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t.replace('"adms:status": { "@id": "mi:ProofOfConcept" }', '"adms:status": { "@id": "http://purl.org/adms/status/Completed" }')), (root) => { + assert.deepEqual(specIssues(root), ['spec/demo/v0.0.1/demo.jsonld: adms:status is not mi:ProofOfConcept — every hosted specification is a proof of concept and must say so in-band']); + }); + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t.replace('"dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }', '"dcterms:license": { "@id": "https://creativecommons.org/licenses/by/4.0/" }')), (root) => { + assert.deepEqual(specIssues(root), ['spec/demo/v0.0.1/demo.jsonld: dcterms:license is not CC0 ()']); + }); +}); + +test('spec check: the @id, version and act must match where the spec is served', () => { + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t.replace('"@id": "https://material-identity.eu/spec/demo/v0.0.1",', '"@id": "https://material-identity.eu/spec/demo/v0.0.2",')), (root) => { + const issues = specIssues(root); + assert.ok(issues[0].includes('no node https://material-identity.eu/spec/demo/v0.0.1 typed mi:ContentSpecification')); + }); + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t.replace('"dcat:version": "v0.0.1"', '"dcat:version": "v9.9.9"').replace('"mi:act": "demo"', '"mi:act": "steel"')), (root) => { + assert.deepEqual(specIssues(root), [ + 'spec/demo/v0.0.1/demo.jsonld: dcat:version is not "v0.0.1", the version directory it is served from', + 'spec/demo/v0.0.1/demo.jsonld: mi:act is not "demo", the act directory it is served from', + ]); + }); +}); + +test('spec check: every pin must be a published entry, carry a digest, and match the served bytes', () => { + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t + .replace('def:4eae703a-fa15-4118-8bc4-7926a22a12fb",\n "checksum"', 'def:00000000-0000-4000-8000-000000000000",\n "checksum"') + .replace('"sha256": "51a28c90bcf24fc116251ef96216a331f278556bf92935dda49f345971cfe075"', '"sha256": "00"') + .replace(/,\n "checksum": \{ "algorithm": "spdx:checksumAlgorithm_sha256", "sha256": "a9ba77a7[0-9a-f]+" \}/, '')), (root) => { + assert.deepEqual(specIssues(root), [ + 'spec/demo/v0.0.1/demo.jsonld: pinned https://material-identity.eu/def/00000000-0000-4000-8000-000000000000 is not an entry in published/', + 'spec/demo/v0.0.1/demo.jsonld: pinned https://material-identity.eu/def/7a1e2c3d-4b5f-4a6e-9c8d-1f2e3d4c5b6a carries no spdx:checksum — a pin without a digest is never publishable', + 'spec/demo/v0.0.1/demo.jsonld: pinned https://material-identity.eu/def/c38a85eb-1a37-416d-ab21-7ddcc599754d has checksum 00, but /def/c38a85eb-1a37-416d-ab21-7ddcc599754d serves 51a28c90bcf24fc116251ef96216a331f278556bf92935dda49f345971cfe075', + ]); + }); + withTree((dir) => edit(dir, 'demo.jsonld', (t) => t.replace(/"pin": \[[\s\S]*?\n \],/, '"pin": [],')), (root) => { + assert.deepEqual(specIssues(root), ['spec/demo/v0.0.1/demo.jsonld: pins no dictionary entry (dcterms:requires → dcat:Distribution with a dcat:downloadURL)']); + }); +}); + +test('spec check: the DCAT record must describe the spec as PoC + CC0, and its checksums must match the hosted bytes', () => { + // a hosted file changed without its record: the digest no longer covers what is served + withTree((dir) => edit(dir, 'full.schema.json', (t) => t.replace('"maxItems": 2', '"maxItems": 3')), (root) => { + const [issue, ...rest] = specIssues(root); + assert.deepEqual(rest, []); + assert.match(issue, /^spec\/demo\/v0\.0\.1\/demo\.dcat\.jsonld: distribution https:\/\/material-identity\.eu\/spec\/demo\/v0\.0\.1\/full\.schema\.json has checksum bb27ef5e[0-9a-f]+, but the hosted full\.schema\.json hashes to [0-9a-f]{64}$/); + }, false); + + withTree((dir) => edit(dir, 'demo.dcat.jsonld', (t) => t + .replace('"adms:status": { "@id": "mi:ProofOfConcept" }', '"adms:status": { "@id": "mi:Final" }') + .replace('"dcterms:license": { "@id": "http://creativecommons.org/publicdomain/zero/1.0/" }', '"dcterms:license": { "@id": "https://example.org/proprietary" }') + .replace('"dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1" }', '"dcat:downloadURL": { "@id": "https://material-identity.eu/spec/demo/v0.0.1/elsewhere.jsonld" }')), (root) => { + assert.deepEqual(specIssues(root), [ + 'spec/demo/v0.0.1/demo.dcat.jsonld: the DCAT record does not mark the spec mi:ProofOfConcept', + 'spec/demo/v0.0.1/demo.dcat.jsonld: the DCAT record\'s dcterms:license is not CC0 ()', + 'spec/demo/v0.0.1/demo.dcat.jsonld: the DCAT record lists no distribution of the spec itself (dcat:downloadURL )', + 'spec/demo/v0.0.1/demo.dcat.jsonld: distribution https://material-identity.eu/spec/demo/v0.0.1/elsewhere.jsonld is not a file hosted for this spec', + ]); + }, false); + + withTree((dir) => writeFileSync(join(dir, 'demo.dcat.jsonld'), '{ "@context": { "dcat": "http://www.w3.org/ns/dcat#" }, "@id": "https://material-identity.eu/catalog", "@type": "dcat:Catalog" }'), (root) => { + const issues = specIssues(root); + assert.ok(issues.includes('spec/demo/v0.0.1/demo.dcat.jsonld: the DCAT record has no node for https://material-identity.eu/spec/demo/v0.0.1')); + assert.ok(issues.includes('spec/demo/v0.0.1/demo.dcat.jsonld: the DCAT record does not describe https://material-identity.eu/spec/demo/v0.0.1 as a dcat:Dataset')); + }, false); +}); + +test('spec check reports layout issues too, so validate fails on a broken hand-over', () => { + withTree((dir) => rmSync(join(dir, 'demo.context.jsonld')), (root) => { + assert.deepEqual(specIssues(root), ['spec/demo/v0.0.1/demo.context.jsonld: missing from the hand-over']); + }, false); +}); + +test('check 1 — spec/ is add-only like published/', () => { + const issues = checkImmutability([ + { status: 'A', path: 'spec/steel/v0.0.3/steel.jsonld' }, + { status: 'M', path: 'spec/steel/v0.0.3/full.schema.json' }, + { status: 'D', path: 'spec/steel/v0.0.3/steel.context.jsonld' }, + { status: 'M', path: 'specs-notes.md' }, + ]); + assert.deepEqual(issues.map((i) => i.file), ['spec/steel/v0.0.3/full.schema.json', 'spec/steel/v0.0.3/steel.context.jsonld']); + assert.match(issues[0].message, /hosted specification file modified — spec\/ is add-only/); + assert.match(issues[1].message, /deleted/); +}); + +// ------------------------------------------------------------------ build + +test('the build serves every spec file byte-identical at the origin path the Worker maps its IRI onto', () => { + const out = mkdtempSync(join(tmpdir(), 'dict-site-')); + try { + build(GREEN, out); + const src = join(GREEN, REL); + const dst = join(out, REL); + assert.deepEqual(readFileSync(join(dst, 'index.jsonld')), readFileSync(join(src, 'demo.jsonld'))); + assert.deepEqual(readFileSync(join(dst, 'context.jsonld')), readFileSync(join(src, 'demo.context.jsonld'))); + assert.deepEqual(readFileSync(join(dst, 'compressed.schema.json')), readFileSync(join(src, 'compressed.schema.json'))); + assert.deepEqual(readFileSync(join(dst, 'full.schema.json')), readFileSync(join(src, 'full.schema.json'))); + + const html = readFileSync(join(dst, 'index.html'), 'utf8'); + assert.ok(html.includes(pocBanner()), 'the spec page carries the PoC banner'); + assert.match(html, //); + assert.match(html, //); + assert.match(html, /

Demo content specification — test fixture \(PoC\)<\/h1>/); + assert.match(html, /

Demo-Inhaltsspezifikation — Testdatensatz \(PoC\)<\/p>/); + assert.match(html, /mi:ProofOfConcept<\/a>/); + assert.match(html, /rel="external license">CC0 1\.0<\/a>/); + assert.match(html, /https:\/\/material-identity\.eu\/spec\/demo<\/a>/); + for (const name of [...SERVED, 'demo.dcat.jsonld']) { + assert.ok(html.includes(`${sha(readFileSync(join(src, name)))}`), `the page shows the SHA-256 of ${name}`); + } + for (const link of ['/spec/demo/v0.0.1/context.jsonld', '/spec/demo/v0.0.1/compressed.schema.json', '/spec/demo/v0.0.1/full.schema.json', '/catalog']) { + assert.ok(html.includes(``), `the page links ${link}`); + } + // pins and memberships in sh:order, nested paths dotted, conditions shown + assert.match(html, /Maximum allowable pressure<\/a><\/td>https:\/\/material-identity\.eu\/def\/c38a85eb-1a37-416d-ab21-7ddcc599754d<\/code><\/td>51a28c90/); + const order = ['mechanicalProperties', 'mechanicalProperties.maxPressure', 'steelmakingRoute'].map((s) => html.indexOf(s)); + assert.ok(order.every((i, k) => i > 0 && (k === 0 || i > order[k - 1])), 'memberships follow sh:order, children under their parent'); + assert.match(html, /optional
Mandatory where the route is known\.<\/span>/); + assert.ok(!/